diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..79c3c705d2660fbd3d6e486c8acf63275c3e9e84 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/onphamd0) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2df4ea7414d0e737473362c9f6bf2ca4c37e1c35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/7xpb8te3) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fa2152b2f74b3e7185266f2093e18c6bb8d95aa1 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-100/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24906600249066002, + "eval_steps": 20, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9823576763965440.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f9090c4876be5a6ff9a7dd919da4c6e0fbaf27a9 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1000/trainer_state.json @@ -0,0 +1,1084 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.488169364881694, + "eval_steps": 20, + "global_step": 1000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + }, + { + "entropy": 0.6744543805718421, + "epoch": 0.298879202988792, + "grad_norm": 0.932099461555481, + "learning_rate": 6.698322232264434e-05, + "loss": 0.5991750717163086, + "mean_token_accuracy": 0.8304223112761975, + "num_tokens": 281241.0, + "step": 120 + }, + { + "epoch": 0.298879202988792, + "eval_entropy": 0.6813044282932614, + "eval_loss": 0.5922021269798279, + "eval_mean_token_accuracy": 0.8346439617317777, + "eval_num_tokens": 281241.0, + "eval_runtime": 86.1551, + "eval_samples_per_second": 15.96, + "eval_steps_per_second": 1.996, + "step": 120 + }, + { + "entropy": 0.6663189359009266, + "epoch": 0.34869240348692404, + "grad_norm": 0.9528499841690063, + "learning_rate": 7.824090674661818e-05, + "loss": 0.5891091346740722, + "mean_token_accuracy": 0.832152470946312, + "num_tokens": 327393.0, + "step": 140 + }, + { + "epoch": 0.34869240348692404, + "eval_entropy": 0.6398407208711602, + "eval_loss": 0.5859636664390564, + "eval_mean_token_accuracy": 0.8372074996316156, + "eval_num_tokens": 327393.0, + "eval_runtime": 86.2706, + "eval_samples_per_second": 15.938, + "eval_steps_per_second": 1.994, + "step": 140 + }, + { + "entropy": 0.64859763905406, + "epoch": 0.398505603985056, + "grad_norm": 0.8468204140663147, + "learning_rate": 8.949859117059201e-05, + "loss": 0.569426441192627, + "mean_token_accuracy": 0.8401990942656994, + "num_tokens": 373834.0, + "step": 160 + }, + { + "epoch": 0.398505603985056, + "eval_entropy": 0.6381674285891444, + "eval_loss": 0.5744525790214539, + "eval_mean_token_accuracy": 0.838626817908398, + "eval_num_tokens": 373834.0, + "eval_runtime": 86.2848, + "eval_samples_per_second": 15.936, + "eval_steps_per_second": 1.993, + "step": 160 + }, + { + "entropy": 0.6432608783245086, + "epoch": 0.44831880448318806, + "grad_norm": 0.8765804767608643, + "learning_rate": 0.00010075627559456587, + "loss": 0.5687318801879883, + "mean_token_accuracy": 0.839249350130558, + "num_tokens": 422572.0, + "step": 180 + }, + { + "epoch": 0.44831880448318806, + "eval_entropy": 0.6047098288355872, + "eval_loss": 0.5679298043251038, + "eval_mean_token_accuracy": 0.8410577181466791, + "eval_num_tokens": 422572.0, + "eval_runtime": 86.5879, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.986, + "step": 180 + }, + { + "entropy": 0.6440276011824608, + "epoch": 0.49813200498132004, + "grad_norm": 0.9576020240783691, + "learning_rate": 0.00011201396001853971, + "loss": 0.5828506469726562, + "mean_token_accuracy": 0.837553184479475, + "num_tokens": 471879.0, + "step": 200 + }, + { + "epoch": 0.49813200498132004, + "eval_entropy": 0.6161119360909906, + "eval_loss": 0.5702911615371704, + "eval_mean_token_accuracy": 0.8407089398350827, + "eval_num_tokens": 471879.0, + "eval_runtime": 86.3341, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 200 + }, + { + "entropy": 0.6377195850014686, + "epoch": 0.547945205479452, + "grad_norm": 0.7212373614311218, + "learning_rate": 0.00012327164444251353, + "loss": 0.5702451229095459, + "mean_token_accuracy": 0.8397969007492065, + "num_tokens": 520984.0, + "step": 220 + }, + { + "epoch": 0.547945205479452, + "eval_entropy": 0.6080108886194784, + "eval_loss": 0.5633499622344971, + "eval_mean_token_accuracy": 0.8396634854549585, + "eval_num_tokens": 520984.0, + "eval_runtime": 86.4945, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 220 + }, + { + "entropy": 0.6287345830351114, + "epoch": 0.597758405977584, + "grad_norm": 0.848779022693634, + "learning_rate": 0.00013452932886648739, + "loss": 0.5506546020507812, + "mean_token_accuracy": 0.8438881888985634, + "num_tokens": 566596.0, + "step": 240 + }, + { + "epoch": 0.597758405977584, + "eval_entropy": 0.6307531505130058, + "eval_loss": 0.5573338270187378, + "eval_mean_token_accuracy": 0.8431362606758295, + "eval_num_tokens": 566596.0, + "eval_runtime": 86.3535, + "eval_samples_per_second": 15.923, + "eval_steps_per_second": 1.992, + "step": 240 + }, + { + "entropy": 0.6223786748945713, + "epoch": 0.6475716064757161, + "grad_norm": 0.7316951751708984, + "learning_rate": 0.0001457870132904612, + "loss": 0.5495625972747803, + "mean_token_accuracy": 0.8440376669168472, + "num_tokens": 613603.0, + "step": 260 + }, + { + "epoch": 0.6475716064757161, + "eval_entropy": 0.623454462476941, + "eval_loss": 0.5619264245033264, + "eval_mean_token_accuracy": 0.8431175777385401, + "eval_num_tokens": 613603.0, + "eval_runtime": 86.2008, + "eval_samples_per_second": 15.951, + "eval_steps_per_second": 1.995, + "step": 260 + }, + { + "entropy": 0.6281675305217505, + "epoch": 0.6973848069738481, + "grad_norm": 0.7639564871788025, + "learning_rate": 0.00015704469771443506, + "loss": 0.5604369163513183, + "mean_token_accuracy": 0.8401600055396556, + "num_tokens": 658565.0, + "step": 280 + }, + { + "epoch": 0.6973848069738481, + "eval_entropy": 0.63416675980701, + "eval_loss": 0.5612760782241821, + "eval_mean_token_accuracy": 0.842435666294985, + "eval_num_tokens": 658565.0, + "eval_runtime": 86.25, + "eval_samples_per_second": 15.942, + "eval_steps_per_second": 1.994, + "step": 280 + }, + { + "entropy": 0.6427909277379513, + "epoch": 0.7471980074719801, + "grad_norm": 0.6475813388824463, + "learning_rate": 0.0001683023821384089, + "loss": 0.573763370513916, + "mean_token_accuracy": 0.8370340794324875, + "num_tokens": 705680.0, + "step": 300 + }, + { + "epoch": 0.7471980074719801, + "eval_entropy": 0.6231539840268534, + "eval_loss": 0.5566866397857666, + "eval_mean_token_accuracy": 0.844177934319474, + "eval_num_tokens": 705680.0, + "eval_runtime": 86.4858, + "eval_samples_per_second": 15.899, + "eval_steps_per_second": 1.989, + "step": 300 + }, + { + "entropy": 0.6226776849478484, + "epoch": 0.797011207970112, + "grad_norm": 0.8886699676513672, + "learning_rate": 0.00017956006656238274, + "loss": 0.558210802078247, + "mean_token_accuracy": 0.84083157107234, + "num_tokens": 752616.0, + "step": 320 + }, + { + "epoch": 0.797011207970112, + "eval_entropy": 0.6066981683983359, + "eval_loss": 0.5585207939147949, + "eval_mean_token_accuracy": 0.8423153311014175, + "eval_num_tokens": 752616.0, + "eval_runtime": 86.3463, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 320 + }, + { + "entropy": 0.6249004438519478, + "epoch": 0.8468244084682441, + "grad_norm": 0.8791211843490601, + "learning_rate": 0.00019081775098635657, + "loss": 0.5603597164154053, + "mean_token_accuracy": 0.8420463085174561, + "num_tokens": 797151.0, + "step": 340 + }, + { + "epoch": 0.8468244084682441, + "eval_entropy": 0.6082247584018596, + "eval_loss": 0.5616299510002136, + "eval_mean_token_accuracy": 0.8431286801432454, + "eval_num_tokens": 797151.0, + "eval_runtime": 86.1253, + "eval_samples_per_second": 15.965, + "eval_steps_per_second": 1.997, + "step": 340 + }, + { + "entropy": 0.6362396612763405, + "epoch": 0.8966376089663761, + "grad_norm": 0.8606319427490234, + "learning_rate": 0.0002020754354103304, + "loss": 0.5735773563385009, + "mean_token_accuracy": 0.8371490836143494, + "num_tokens": 843585.0, + "step": 360 + }, + { + "epoch": 0.8966376089663761, + "eval_entropy": 0.6492362072648004, + "eval_loss": 0.5646467804908752, + "eval_mean_token_accuracy": 0.8415517574825953, + "eval_num_tokens": 843585.0, + "eval_runtime": 86.3351, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 360 + }, + { + "entropy": 0.638665035739541, + "epoch": 0.9464508094645081, + "grad_norm": 0.7773950099945068, + "learning_rate": 0.00021333311983430425, + "loss": 0.5820859909057617, + "mean_token_accuracy": 0.8372561208903789, + "num_tokens": 889842.0, + "step": 380 + }, + { + "epoch": 0.9464508094645081, + "eval_entropy": 0.6434498637221581, + "eval_loss": 0.5645168423652649, + "eval_mean_token_accuracy": 0.8420382481674815, + "eval_num_tokens": 889842.0, + "eval_runtime": 86.1216, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 380 + }, + { + "entropy": 0.6316851265728474, + "epoch": 0.9962640099626401, + "grad_norm": 1.6120579242706299, + "learning_rate": 0.00022459080425827807, + "loss": 0.5637502670288086, + "mean_token_accuracy": 0.8386227294802666, + "num_tokens": 935589.0, + "step": 400 + }, + { + "epoch": 0.9962640099626401, + "eval_entropy": 0.6469012776086497, + "eval_loss": 0.5758090615272522, + "eval_mean_token_accuracy": 0.8397158470957778, + "eval_num_tokens": 935589.0, + "eval_runtime": 86.6139, + "eval_samples_per_second": 15.875, + "eval_steps_per_second": 1.986, + "step": 400 + }, + { + "entropy": 0.5894816922835815, + "epoch": 1.0448318804483188, + "grad_norm": 1.1616325378417969, + "learning_rate": 0.00022626713048053178, + "loss": 0.5316025257110596, + "mean_token_accuracy": 0.8466163017810919, + "num_tokens": 980589.0, + "step": 420 + }, + { + "epoch": 1.0448318804483188, + "eval_entropy": 0.5860798164855602, + "eval_loss": 0.5777581930160522, + "eval_mean_token_accuracy": 0.8396938103576039, + "eval_num_tokens": 980589.0, + "eval_runtime": 86.1449, + "eval_samples_per_second": 15.961, + "eval_steps_per_second": 1.997, + "step": 420 + }, + { + "entropy": 0.5818420693278312, + "epoch": 1.0946450809464507, + "grad_norm": 0.7999453544616699, + "learning_rate": 0.00022622107023288778, + "loss": 0.5221010208129883, + "mean_token_accuracy": 0.8474301159381866, + "num_tokens": 1027852.0, + "step": 440 + }, + { + "epoch": 1.0946450809464507, + "eval_entropy": 0.5783926014636838, + "eval_loss": 0.5700300931930542, + "eval_mean_token_accuracy": 0.8430753537388735, + "eval_num_tokens": 1027852.0, + "eval_runtime": 86.5308, + "eval_samples_per_second": 15.89, + "eval_steps_per_second": 1.988, + "step": 440 + }, + { + "entropy": 0.5612493887543678, + "epoch": 1.1444582814445827, + "grad_norm": 1.015687346458435, + "learning_rate": 0.00022614090619491568, + "loss": 0.5084867000579834, + "mean_token_accuracy": 0.8495561093091964, + "num_tokens": 1077649.0, + "step": 460 + }, + { + "epoch": 1.1444582814445827, + "eval_entropy": 0.5841563874205877, + "eval_loss": 0.5693665742874146, + "eval_mean_token_accuracy": 0.8427817298229351, + "eval_num_tokens": 1077649.0, + "eval_runtime": 86.5256, + "eval_samples_per_second": 15.891, + "eval_steps_per_second": 1.988, + "step": 460 + }, + { + "entropy": 0.5828216474503278, + "epoch": 1.1942714819427147, + "grad_norm": 1.9750930070877075, + "learning_rate": 0.00022602666254299594, + "loss": 0.5180017948150635, + "mean_token_accuracy": 0.8515685826539994, + "num_tokens": 1124872.0, + "step": 480 + }, + { + "epoch": 1.1942714819427147, + "eval_entropy": 0.5806607044366903, + "eval_loss": 0.5804352760314941, + "eval_mean_token_accuracy": 0.8413014668364858, + "eval_num_tokens": 1124872.0, + "eval_runtime": 86.1199, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 480 + }, + { + "entropy": 0.5926914308220148, + "epoch": 1.244084682440847, + "grad_norm": 0.8917353749275208, + "learning_rate": 0.0002258783737314558, + "loss": 0.528910779953003, + "mean_token_accuracy": 0.8486074328422546, + "num_tokens": 1168698.0, + "step": 500 + }, + { + "epoch": 1.244084682440847, + "eval_entropy": 0.5593361884009006, + "eval_loss": 0.5675153732299805, + "eval_mean_token_accuracy": 0.8433507802181466, + "eval_num_tokens": 1168698.0, + "eval_runtime": 86.7289, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 500 + }, + { + "entropy": 0.5865630559623242, + "epoch": 1.293897882938979, + "grad_norm": 0.7482362985610962, + "learning_rate": 0.00022569608448217823, + "loss": 0.5250466823577881, + "mean_token_accuracy": 0.8477916084229946, + "num_tokens": 1216679.0, + "step": 520 + }, + { + "epoch": 1.293897882938979, + "eval_entropy": 0.543057840230853, + "eval_loss": 0.5671008229255676, + "eval_mean_token_accuracy": 0.8428726016088973, + "eval_num_tokens": 1216679.0, + "eval_runtime": 86.3403, + "eval_samples_per_second": 15.925, + "eval_steps_per_second": 1.992, + "step": 520 + }, + { + "entropy": 0.5870206747204065, + "epoch": 1.3437110834371109, + "grad_norm": 0.9473814964294434, + "learning_rate": 0.00022547984977111448, + "loss": 0.5252370834350586, + "mean_token_accuracy": 0.8468369916081429, + "num_tokens": 1261365.0, + "step": 540 + }, + { + "epoch": 1.3437110834371109, + "eval_entropy": 0.590982622878496, + "eval_loss": 0.5676343441009521, + "eval_mean_token_accuracy": 0.8429348746011424, + "eval_num_tokens": 1261365.0, + "eval_runtime": 86.5168, + "eval_samples_per_second": 15.893, + "eval_steps_per_second": 1.988, + "step": 540 + }, + { + "entropy": 0.5785854265093804, + "epoch": 1.3935242839352429, + "grad_norm": 0.9353351593017578, + "learning_rate": 0.0002252297348117042, + "loss": 0.5304938316345215, + "mean_token_accuracy": 0.8463383808732032, + "num_tokens": 1306879.0, + "step": 560 + }, + { + "epoch": 1.3935242839352429, + "eval_entropy": 0.6099918867612995, + "eval_loss": 0.5620437860488892, + "eval_mean_token_accuracy": 0.8430728347495545, + "eval_num_tokens": 1306879.0, + "eval_runtime": 86.7741, + "eval_samples_per_second": 15.846, + "eval_steps_per_second": 1.982, + "step": 560 + }, + { + "entropy": 0.5768801040947438, + "epoch": 1.4433374844333748, + "grad_norm": 0.9198738932609558, + "learning_rate": 0.0002249458150352077, + "loss": 0.520513391494751, + "mean_token_accuracy": 0.8487689301371575, + "num_tokens": 1353534.0, + "step": 580 + }, + { + "epoch": 1.4433374844333748, + "eval_entropy": 0.6349420670506566, + "eval_loss": 0.5645340085029602, + "eval_mean_token_accuracy": 0.8447844597489335, + "eval_num_tokens": 1353534.0, + "eval_runtime": 86.3257, + "eval_samples_per_second": 15.928, + "eval_steps_per_second": 1.992, + "step": 580 + }, + { + "entropy": 0.5822233572602272, + "epoch": 1.4931506849315068, + "grad_norm": 0.832811176776886, + "learning_rate": 0.0002246281760679571, + "loss": 0.5295282363891601, + "mean_token_accuracy": 0.8504064798355102, + "num_tokens": 1399537.0, + "step": 600 + }, + { + "epoch": 1.4931506849315068, + "eval_entropy": 0.5829724387027496, + "eval_loss": 0.5612193942070007, + "eval_mean_token_accuracy": 0.8449643853791925, + "eval_num_tokens": 1399537.0, + "eval_runtime": 86.6617, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 600 + }, + { + "entropy": 0.571855777129531, + "epoch": 1.5429638854296388, + "grad_norm": 0.7665547728538513, + "learning_rate": 0.00022427691370553263, + "loss": 0.5187931060791016, + "mean_token_accuracy": 0.8534420043230057, + "num_tokens": 1448422.0, + "step": 620 + }, + { + "epoch": 1.5429638854296388, + "eval_entropy": 0.5623592240519302, + "eval_loss": 0.5575760006904602, + "eval_mean_token_accuracy": 0.8468210229346919, + "eval_num_tokens": 1448422.0, + "eval_runtime": 86.6324, + "eval_samples_per_second": 15.872, + "eval_steps_per_second": 1.985, + "step": 620 + }, + { + "entropy": 0.5740394659340382, + "epoch": 1.592777085927771, + "grad_norm": 0.6500429511070251, + "learning_rate": 0.00022389213388387174, + "loss": 0.5283198833465577, + "mean_token_accuracy": 0.8502798482775689, + "num_tokens": 1495009.0, + "step": 640 + }, + { + "epoch": 1.592777085927771, + "eval_entropy": 0.5548852207355721, + "eval_loss": 0.5561797022819519, + "eval_mean_token_accuracy": 0.8452786498291548, + "eval_num_tokens": 1495009.0, + "eval_runtime": 86.5205, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 640 + }, + { + "entropy": 0.6020145989954472, + "epoch": 1.6425902864259028, + "grad_norm": 0.7056867480278015, + "learning_rate": 0.00022347395264732053, + "loss": 0.5400049209594726, + "mean_token_accuracy": 0.8447613954544068, + "num_tokens": 1536932.0, + "step": 660 + }, + { + "epoch": 1.6425902864259028, + "eval_entropy": 0.5618055154417836, + "eval_loss": 0.556106686592102, + "eval_mean_token_accuracy": 0.8465680112672407, + "eval_num_tokens": 1536932.0, + "eval_runtime": 86.2971, + "eval_samples_per_second": 15.933, + "eval_steps_per_second": 1.993, + "step": 660 + }, + { + "entropy": 0.5665927153080702, + "epoch": 1.692403486924035, + "grad_norm": 0.5987663865089417, + "learning_rate": 0.00022302249611363625, + "loss": 0.5143643856048584, + "mean_token_accuracy": 0.8529589556157589, + "num_tokens": 1585718.0, + "step": 680 + }, + { + "epoch": 1.692403486924035, + "eval_entropy": 0.568248552118623, + "eval_loss": 0.5476346015930176, + "eval_mean_token_accuracy": 0.8476775434128073, + "eval_num_tokens": 1585718.0, + "eval_runtime": 86.9583, + "eval_samples_per_second": 15.812, + "eval_steps_per_second": 1.978, + "step": 680 + }, + { + "entropy": 0.5673687808215618, + "epoch": 1.7422166874221667, + "grad_norm": 0.735261857509613, + "learning_rate": 0.00022253790043595193, + "loss": 0.509885597229004, + "mean_token_accuracy": 0.8537046857178211, + "num_tokens": 1635718.0, + "step": 700 + }, + { + "epoch": 1.7422166874221667, + "eval_entropy": 0.5616967284748721, + "eval_loss": 0.5439274311065674, + "eval_mean_token_accuracy": 0.8488946217437123, + "eval_num_tokens": 1635718.0, + "eval_runtime": 86.0604, + "eval_samples_per_second": 15.977, + "eval_steps_per_second": 1.999, + "step": 700 + }, + { + "entropy": 0.5529541682451964, + "epoch": 1.792029887920299, + "grad_norm": 0.7014835476875305, + "learning_rate": 0.00022202031176171442, + "loss": 0.5078992366790771, + "mean_token_accuracy": 0.8525233261287213, + "num_tokens": 1681291.0, + "step": 720 + }, + { + "epoch": 1.792029887920299, + "eval_entropy": 0.5827173320359962, + "eval_loss": 0.5419450402259827, + "eval_mean_token_accuracy": 0.8477318609176681, + "eval_num_tokens": 1681291.0, + "eval_runtime": 85.2984, + "eval_samples_per_second": 16.12, + "eval_steps_per_second": 2.016, + "step": 720 + }, + { + "entropy": 0.5755720350891351, + "epoch": 1.841843088418431, + "grad_norm": 0.705613911151886, + "learning_rate": 0.00022146988618860824, + "loss": 0.5181350708007812, + "mean_token_accuracy": 0.8467609457671642, + "num_tokens": 1729102.0, + "step": 740 + }, + { + "epoch": 1.841843088418431, + "eval_entropy": 0.5743971356125765, + "eval_loss": 0.5415896773338318, + "eval_mean_token_accuracy": 0.847328585940738, + "eval_num_tokens": 1729102.0, + "eval_runtime": 85.5602, + "eval_samples_per_second": 16.071, + "eval_steps_per_second": 2.01, + "step": 740 + }, + { + "entropy": 0.561330484598875, + "epoch": 1.891656288916563, + "grad_norm": 0.6722865700721741, + "learning_rate": 0.0002208867897174789, + "loss": 0.499837589263916, + "mean_token_accuracy": 0.8518734864890576, + "num_tokens": 1773578.0, + "step": 760 + }, + { + "epoch": 1.891656288916563, + "eval_entropy": 0.5865232653396074, + "eval_loss": 0.5437926650047302, + "eval_mean_token_accuracy": 0.8450997017843779, + "eval_num_tokens": 1773578.0, + "eval_runtime": 86.4116, + "eval_samples_per_second": 15.912, + "eval_steps_per_second": 1.99, + "step": 760 + }, + { + "entropy": 0.547389242425561, + "epoch": 1.9414694894146949, + "grad_norm": 0.7935577034950256, + "learning_rate": 0.00022027119820226907, + "loss": 0.4977591514587402, + "mean_token_accuracy": 0.8539491161704064, + "num_tokens": 1821725.0, + "step": 780 + }, + { + "epoch": 1.9414694894146949, + "eval_entropy": 0.5290903090391048, + "eval_loss": 0.5409526824951172, + "eval_mean_token_accuracy": 0.8497545698354411, + "eval_num_tokens": 1821725.0, + "eval_runtime": 86.7262, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 780 + }, + { + "entropy": 0.5687909748405218, + "epoch": 1.9912826899128269, + "grad_norm": 0.6180546283721924, + "learning_rate": 0.00021962329729698345, + "loss": 0.5109643459320068, + "mean_token_accuracy": 0.8521598495543004, + "num_tokens": 1868431.0, + "step": 800 + }, + { + "epoch": 1.9912826899128269, + "eval_entropy": 0.5503541858390321, + "eval_loss": 0.5361555218696594, + "eval_mean_token_accuracy": 0.8510884285666221, + "eval_num_tokens": 1868431.0, + "eval_runtime": 86.3339, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 800 + }, + { + "entropy": 0.4739728841261986, + "epoch": 2.0398505603985058, + "grad_norm": 0.8058829307556152, + "learning_rate": 0.0002189432823996982, + "loss": 0.4204097747802734, + "mean_token_accuracy": 0.8728981889211215, + "num_tokens": 1915280.0, + "step": 820 + }, + { + "epoch": 2.0398505603985058, + "eval_entropy": 0.5077334992414297, + "eval_loss": 0.5531114339828491, + "eval_mean_token_accuracy": 0.8489257208136625, + "eval_num_tokens": 1915280.0, + "eval_runtime": 86.4801, + "eval_samples_per_second": 15.9, + "eval_steps_per_second": 1.989, + "step": 820 + }, + { + "entropy": 0.4594309840351343, + "epoch": 2.0896637608966375, + "grad_norm": 0.6906896829605103, + "learning_rate": 0.0002182313585936314, + "loss": 0.4071959495544434, + "mean_token_accuracy": 0.8732857562601566, + "num_tokens": 1965306.0, + "step": 840 + }, + { + "epoch": 2.0896637608966375, + "eval_entropy": 0.49850136994622474, + "eval_loss": 0.5486204624176025, + "eval_mean_token_accuracy": 0.8507991450470548, + "eval_num_tokens": 1965306.0, + "eval_runtime": 86.3364, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 840 + }, + { + "entropy": 0.4881629109382629, + "epoch": 2.1394769613947697, + "grad_norm": 0.6343470215797424, + "learning_rate": 0.0002174877405852928, + "loss": 0.41669540405273436, + "mean_token_accuracy": 0.8711295068264008, + "num_tokens": 2008562.0, + "step": 860 + }, + { + "epoch": 2.1394769613947697, + "eval_entropy": 0.49155513924914734, + "eval_loss": 0.555109441280365, + "eval_mean_token_accuracy": 0.8496399400539176, + "eval_num_tokens": 2008562.0, + "eval_runtime": 86.3295, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 860 + }, + { + "entropy": 0.4648668970912695, + "epoch": 2.1892901618929015, + "grad_norm": 0.8014165163040161, + "learning_rate": 0.00021671265263973133, + "loss": 0.4110250473022461, + "mean_token_accuracy": 0.8754166305065155, + "num_tokens": 2056474.0, + "step": 880 + }, + { + "epoch": 2.1892901618929015, + "eval_entropy": 0.4909258722219356, + "eval_loss": 0.5539511442184448, + "eval_mean_token_accuracy": 0.8492401502160138, + "eval_num_tokens": 2056474.0, + "eval_runtime": 86.3468, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 880 + }, + { + "entropy": 0.4824485514312983, + "epoch": 2.2391033623910337, + "grad_norm": 0.6665191054344177, + "learning_rate": 0.00021590632851289967, + "loss": 0.4181404113769531, + "mean_token_accuracy": 0.8726993151009083, + "num_tokens": 2103543.0, + "step": 900 + }, + { + "epoch": 2.2391033623910337, + "eval_entropy": 0.4986876940657926, + "eval_loss": 0.547695517539978, + "eval_mean_token_accuracy": 0.8501384708770486, + "eval_num_tokens": 2103543.0, + "eval_runtime": 86.3838, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 900 + }, + { + "entropy": 0.4751896943897009, + "epoch": 2.2889165628891655, + "grad_norm": 0.81158047914505, + "learning_rate": 0.00021506901138115678, + "loss": 0.40689678192138673, + "mean_token_accuracy": 0.8745221219956875, + "num_tokens": 2147861.0, + "step": 920 + }, + { + "epoch": 2.2889165628891655, + "eval_entropy": 0.507153491121392, + "eval_loss": 0.5501641631126404, + "eval_mean_token_accuracy": 0.8495670116918032, + "eval_num_tokens": 2147861.0, + "eval_runtime": 86.0912, + "eval_samples_per_second": 15.971, + "eval_steps_per_second": 1.998, + "step": 920 + }, + { + "entropy": 0.4873133715242147, + "epoch": 2.3387297633872977, + "grad_norm": 0.7218056321144104, + "learning_rate": 0.0002142009537679292, + "loss": 0.42701358795166017, + "mean_token_accuracy": 0.8695114746689796, + "num_tokens": 2190561.0, + "step": 940 + }, + { + "epoch": 2.3387297633872977, + "eval_entropy": 0.5202612736543943, + "eval_loss": 0.5491839051246643, + "eval_mean_token_accuracy": 0.8494071208460386, + "eval_num_tokens": 2190561.0, + "eval_runtime": 86.1142, + "eval_samples_per_second": 15.967, + "eval_steps_per_second": 1.997, + "step": 940 + }, + { + "entropy": 0.4762951169162989, + "epoch": 2.3885429638854294, + "grad_norm": 0.7194424867630005, + "learning_rate": 0.0002133024174675534, + "loss": 0.42299847602844237, + "mean_token_accuracy": 0.8709790132939815, + "num_tokens": 2239412.0, + "step": 960 + }, + { + "epoch": 2.3885429638854294, + "eval_entropy": 0.4899340462546016, + "eval_loss": 0.5522511601448059, + "eval_mean_token_accuracy": 0.8492208258357159, + "eval_num_tokens": 2239412.0, + "eval_runtime": 86.463, + "eval_samples_per_second": 15.903, + "eval_steps_per_second": 1.989, + "step": 960 + }, + { + "entropy": 0.49650347977876663, + "epoch": 2.4383561643835616, + "grad_norm": 0.8406022787094116, + "learning_rate": 0.0002123736734663221, + "loss": 0.4275330066680908, + "mean_token_accuracy": 0.8670595556497573, + "num_tokens": 2286283.0, + "step": 980 + }, + { + "epoch": 2.4383561643835616, + "eval_entropy": 0.49691385654515996, + "eval_loss": 0.5491269826889038, + "eval_mean_token_accuracy": 0.850309816210769, + "eval_num_tokens": 2286283.0, + "eval_runtime": 86.17, + "eval_samples_per_second": 15.957, + "eval_steps_per_second": 1.996, + "step": 980 + }, + { + "entropy": 0.48843890577554705, + "epoch": 2.488169364881694, + "grad_norm": 0.9082473516464233, + "learning_rate": 0.00021141500186075868, + "loss": 0.4309722423553467, + "mean_token_accuracy": 0.8686766296625137, + "num_tokens": 2333733.0, + "step": 1000 + }, + { + "epoch": 2.488169364881694, + "eval_entropy": 0.5543508351195691, + "eval_loss": 0.5478800535202026, + "eval_mean_token_accuracy": 0.8478029522784921, + "eval_num_tokens": 2333733.0, + "eval_runtime": 86.3835, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 1000 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.859037950771814e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1f5af10f1037b6f54cf86ad5b92625168c28243d --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1020/trainer_state.json @@ -0,0 +1,1105 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.5379825653798256, + "eval_steps": 20, + "global_step": 1020, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + }, + { + "entropy": 0.6744543805718421, + "epoch": 0.298879202988792, + "grad_norm": 0.932099461555481, + "learning_rate": 6.698322232264434e-05, + "loss": 0.5991750717163086, + "mean_token_accuracy": 0.8304223112761975, + "num_tokens": 281241.0, + "step": 120 + }, + { + "epoch": 0.298879202988792, + "eval_entropy": 0.6813044282932614, + "eval_loss": 0.5922021269798279, + "eval_mean_token_accuracy": 0.8346439617317777, + "eval_num_tokens": 281241.0, + "eval_runtime": 86.1551, + "eval_samples_per_second": 15.96, + "eval_steps_per_second": 1.996, + "step": 120 + }, + { + "entropy": 0.6663189359009266, + "epoch": 0.34869240348692404, + "grad_norm": 0.9528499841690063, + "learning_rate": 7.824090674661818e-05, + "loss": 0.5891091346740722, + "mean_token_accuracy": 0.832152470946312, + "num_tokens": 327393.0, + "step": 140 + }, + { + "epoch": 0.34869240348692404, + "eval_entropy": 0.6398407208711602, + "eval_loss": 0.5859636664390564, + "eval_mean_token_accuracy": 0.8372074996316156, + "eval_num_tokens": 327393.0, + "eval_runtime": 86.2706, + "eval_samples_per_second": 15.938, + "eval_steps_per_second": 1.994, + "step": 140 + }, + { + "entropy": 0.64859763905406, + "epoch": 0.398505603985056, + "grad_norm": 0.8468204140663147, + "learning_rate": 8.949859117059201e-05, + "loss": 0.569426441192627, + "mean_token_accuracy": 0.8401990942656994, + "num_tokens": 373834.0, + "step": 160 + }, + { + "epoch": 0.398505603985056, + "eval_entropy": 0.6381674285891444, + "eval_loss": 0.5744525790214539, + "eval_mean_token_accuracy": 0.838626817908398, + "eval_num_tokens": 373834.0, + "eval_runtime": 86.2848, + "eval_samples_per_second": 15.936, + "eval_steps_per_second": 1.993, + "step": 160 + }, + { + "entropy": 0.6432608783245086, + "epoch": 0.44831880448318806, + "grad_norm": 0.8765804767608643, + "learning_rate": 0.00010075627559456587, + "loss": 0.5687318801879883, + "mean_token_accuracy": 0.839249350130558, + "num_tokens": 422572.0, + "step": 180 + }, + { + "epoch": 0.44831880448318806, + "eval_entropy": 0.6047098288355872, + "eval_loss": 0.5679298043251038, + "eval_mean_token_accuracy": 0.8410577181466791, + "eval_num_tokens": 422572.0, + "eval_runtime": 86.5879, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.986, + "step": 180 + }, + { + "entropy": 0.6440276011824608, + "epoch": 0.49813200498132004, + "grad_norm": 0.9576020240783691, + "learning_rate": 0.00011201396001853971, + "loss": 0.5828506469726562, + "mean_token_accuracy": 0.837553184479475, + "num_tokens": 471879.0, + "step": 200 + }, + { + "epoch": 0.49813200498132004, + "eval_entropy": 0.6161119360909906, + "eval_loss": 0.5702911615371704, + "eval_mean_token_accuracy": 0.8407089398350827, + "eval_num_tokens": 471879.0, + "eval_runtime": 86.3341, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 200 + }, + { + "entropy": 0.6377195850014686, + "epoch": 0.547945205479452, + "grad_norm": 0.7212373614311218, + "learning_rate": 0.00012327164444251353, + "loss": 0.5702451229095459, + "mean_token_accuracy": 0.8397969007492065, + "num_tokens": 520984.0, + "step": 220 + }, + { + "epoch": 0.547945205479452, + "eval_entropy": 0.6080108886194784, + "eval_loss": 0.5633499622344971, + "eval_mean_token_accuracy": 0.8396634854549585, + "eval_num_tokens": 520984.0, + "eval_runtime": 86.4945, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 220 + }, + { + "entropy": 0.6287345830351114, + "epoch": 0.597758405977584, + "grad_norm": 0.848779022693634, + "learning_rate": 0.00013452932886648739, + "loss": 0.5506546020507812, + "mean_token_accuracy": 0.8438881888985634, + "num_tokens": 566596.0, + "step": 240 + }, + { + "epoch": 0.597758405977584, + "eval_entropy": 0.6307531505130058, + "eval_loss": 0.5573338270187378, + "eval_mean_token_accuracy": 0.8431362606758295, + "eval_num_tokens": 566596.0, + "eval_runtime": 86.3535, + "eval_samples_per_second": 15.923, + "eval_steps_per_second": 1.992, + "step": 240 + }, + { + "entropy": 0.6223786748945713, + "epoch": 0.6475716064757161, + "grad_norm": 0.7316951751708984, + "learning_rate": 0.0001457870132904612, + "loss": 0.5495625972747803, + "mean_token_accuracy": 0.8440376669168472, + "num_tokens": 613603.0, + "step": 260 + }, + { + "epoch": 0.6475716064757161, + "eval_entropy": 0.623454462476941, + "eval_loss": 0.5619264245033264, + "eval_mean_token_accuracy": 0.8431175777385401, + "eval_num_tokens": 613603.0, + "eval_runtime": 86.2008, + "eval_samples_per_second": 15.951, + "eval_steps_per_second": 1.995, + "step": 260 + }, + { + "entropy": 0.6281675305217505, + "epoch": 0.6973848069738481, + "grad_norm": 0.7639564871788025, + "learning_rate": 0.00015704469771443506, + "loss": 0.5604369163513183, + "mean_token_accuracy": 0.8401600055396556, + "num_tokens": 658565.0, + "step": 280 + }, + { + "epoch": 0.6973848069738481, + "eval_entropy": 0.63416675980701, + "eval_loss": 0.5612760782241821, + "eval_mean_token_accuracy": 0.842435666294985, + "eval_num_tokens": 658565.0, + "eval_runtime": 86.25, + "eval_samples_per_second": 15.942, + "eval_steps_per_second": 1.994, + "step": 280 + }, + { + "entropy": 0.6427909277379513, + "epoch": 0.7471980074719801, + "grad_norm": 0.6475813388824463, + "learning_rate": 0.0001683023821384089, + "loss": 0.573763370513916, + "mean_token_accuracy": 0.8370340794324875, + "num_tokens": 705680.0, + "step": 300 + }, + { + "epoch": 0.7471980074719801, + "eval_entropy": 0.6231539840268534, + "eval_loss": 0.5566866397857666, + "eval_mean_token_accuracy": 0.844177934319474, + "eval_num_tokens": 705680.0, + "eval_runtime": 86.4858, + "eval_samples_per_second": 15.899, + "eval_steps_per_second": 1.989, + "step": 300 + }, + { + "entropy": 0.6226776849478484, + "epoch": 0.797011207970112, + "grad_norm": 0.8886699676513672, + "learning_rate": 0.00017956006656238274, + "loss": 0.558210802078247, + "mean_token_accuracy": 0.84083157107234, + "num_tokens": 752616.0, + "step": 320 + }, + { + "epoch": 0.797011207970112, + "eval_entropy": 0.6066981683983359, + "eval_loss": 0.5585207939147949, + "eval_mean_token_accuracy": 0.8423153311014175, + "eval_num_tokens": 752616.0, + "eval_runtime": 86.3463, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 320 + }, + { + "entropy": 0.6249004438519478, + "epoch": 0.8468244084682441, + "grad_norm": 0.8791211843490601, + "learning_rate": 0.00019081775098635657, + "loss": 0.5603597164154053, + "mean_token_accuracy": 0.8420463085174561, + "num_tokens": 797151.0, + "step": 340 + }, + { + "epoch": 0.8468244084682441, + "eval_entropy": 0.6082247584018596, + "eval_loss": 0.5616299510002136, + "eval_mean_token_accuracy": 0.8431286801432454, + "eval_num_tokens": 797151.0, + "eval_runtime": 86.1253, + "eval_samples_per_second": 15.965, + "eval_steps_per_second": 1.997, + "step": 340 + }, + { + "entropy": 0.6362396612763405, + "epoch": 0.8966376089663761, + "grad_norm": 0.8606319427490234, + "learning_rate": 0.0002020754354103304, + "loss": 0.5735773563385009, + "mean_token_accuracy": 0.8371490836143494, + "num_tokens": 843585.0, + "step": 360 + }, + { + "epoch": 0.8966376089663761, + "eval_entropy": 0.6492362072648004, + "eval_loss": 0.5646467804908752, + "eval_mean_token_accuracy": 0.8415517574825953, + "eval_num_tokens": 843585.0, + "eval_runtime": 86.3351, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 360 + }, + { + "entropy": 0.638665035739541, + "epoch": 0.9464508094645081, + "grad_norm": 0.7773950099945068, + "learning_rate": 0.00021333311983430425, + "loss": 0.5820859909057617, + "mean_token_accuracy": 0.8372561208903789, + "num_tokens": 889842.0, + "step": 380 + }, + { + "epoch": 0.9464508094645081, + "eval_entropy": 0.6434498637221581, + "eval_loss": 0.5645168423652649, + "eval_mean_token_accuracy": 0.8420382481674815, + "eval_num_tokens": 889842.0, + "eval_runtime": 86.1216, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 380 + }, + { + "entropy": 0.6316851265728474, + "epoch": 0.9962640099626401, + "grad_norm": 1.6120579242706299, + "learning_rate": 0.00022459080425827807, + "loss": 0.5637502670288086, + "mean_token_accuracy": 0.8386227294802666, + "num_tokens": 935589.0, + "step": 400 + }, + { + "epoch": 0.9962640099626401, + "eval_entropy": 0.6469012776086497, + "eval_loss": 0.5758090615272522, + "eval_mean_token_accuracy": 0.8397158470957778, + "eval_num_tokens": 935589.0, + "eval_runtime": 86.6139, + "eval_samples_per_second": 15.875, + "eval_steps_per_second": 1.986, + "step": 400 + }, + { + "entropy": 0.5894816922835815, + "epoch": 1.0448318804483188, + "grad_norm": 1.1616325378417969, + "learning_rate": 0.00022626713048053178, + "loss": 0.5316025257110596, + "mean_token_accuracy": 0.8466163017810919, + "num_tokens": 980589.0, + "step": 420 + }, + { + "epoch": 1.0448318804483188, + "eval_entropy": 0.5860798164855602, + "eval_loss": 0.5777581930160522, + "eval_mean_token_accuracy": 0.8396938103576039, + "eval_num_tokens": 980589.0, + "eval_runtime": 86.1449, + "eval_samples_per_second": 15.961, + "eval_steps_per_second": 1.997, + "step": 420 + }, + { + "entropy": 0.5818420693278312, + "epoch": 1.0946450809464507, + "grad_norm": 0.7999453544616699, + "learning_rate": 0.00022622107023288778, + "loss": 0.5221010208129883, + "mean_token_accuracy": 0.8474301159381866, + "num_tokens": 1027852.0, + "step": 440 + }, + { + "epoch": 1.0946450809464507, + "eval_entropy": 0.5783926014636838, + "eval_loss": 0.5700300931930542, + "eval_mean_token_accuracy": 0.8430753537388735, + "eval_num_tokens": 1027852.0, + "eval_runtime": 86.5308, + "eval_samples_per_second": 15.89, + "eval_steps_per_second": 1.988, + "step": 440 + }, + { + "entropy": 0.5612493887543678, + "epoch": 1.1444582814445827, + "grad_norm": 1.015687346458435, + "learning_rate": 0.00022614090619491568, + "loss": 0.5084867000579834, + "mean_token_accuracy": 0.8495561093091964, + "num_tokens": 1077649.0, + "step": 460 + }, + { + "epoch": 1.1444582814445827, + "eval_entropy": 0.5841563874205877, + "eval_loss": 0.5693665742874146, + "eval_mean_token_accuracy": 0.8427817298229351, + "eval_num_tokens": 1077649.0, + "eval_runtime": 86.5256, + "eval_samples_per_second": 15.891, + "eval_steps_per_second": 1.988, + "step": 460 + }, + { + "entropy": 0.5828216474503278, + "epoch": 1.1942714819427147, + "grad_norm": 1.9750930070877075, + "learning_rate": 0.00022602666254299594, + "loss": 0.5180017948150635, + "mean_token_accuracy": 0.8515685826539994, + "num_tokens": 1124872.0, + "step": 480 + }, + { + "epoch": 1.1942714819427147, + "eval_entropy": 0.5806607044366903, + "eval_loss": 0.5804352760314941, + "eval_mean_token_accuracy": 0.8413014668364858, + "eval_num_tokens": 1124872.0, + "eval_runtime": 86.1199, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 480 + }, + { + "entropy": 0.5926914308220148, + "epoch": 1.244084682440847, + "grad_norm": 0.8917353749275208, + "learning_rate": 0.0002258783737314558, + "loss": 0.528910779953003, + "mean_token_accuracy": 0.8486074328422546, + "num_tokens": 1168698.0, + "step": 500 + }, + { + "epoch": 1.244084682440847, + "eval_entropy": 0.5593361884009006, + "eval_loss": 0.5675153732299805, + "eval_mean_token_accuracy": 0.8433507802181466, + "eval_num_tokens": 1168698.0, + "eval_runtime": 86.7289, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 500 + }, + { + "entropy": 0.5865630559623242, + "epoch": 1.293897882938979, + "grad_norm": 0.7482362985610962, + "learning_rate": 0.00022569608448217823, + "loss": 0.5250466823577881, + "mean_token_accuracy": 0.8477916084229946, + "num_tokens": 1216679.0, + "step": 520 + }, + { + "epoch": 1.293897882938979, + "eval_entropy": 0.543057840230853, + "eval_loss": 0.5671008229255676, + "eval_mean_token_accuracy": 0.8428726016088973, + "eval_num_tokens": 1216679.0, + "eval_runtime": 86.3403, + "eval_samples_per_second": 15.925, + "eval_steps_per_second": 1.992, + "step": 520 + }, + { + "entropy": 0.5870206747204065, + "epoch": 1.3437110834371109, + "grad_norm": 0.9473814964294434, + "learning_rate": 0.00022547984977111448, + "loss": 0.5252370834350586, + "mean_token_accuracy": 0.8468369916081429, + "num_tokens": 1261365.0, + "step": 540 + }, + { + "epoch": 1.3437110834371109, + "eval_entropy": 0.590982622878496, + "eval_loss": 0.5676343441009521, + "eval_mean_token_accuracy": 0.8429348746011424, + "eval_num_tokens": 1261365.0, + "eval_runtime": 86.5168, + "eval_samples_per_second": 15.893, + "eval_steps_per_second": 1.988, + "step": 540 + }, + { + "entropy": 0.5785854265093804, + "epoch": 1.3935242839352429, + "grad_norm": 0.9353351593017578, + "learning_rate": 0.0002252297348117042, + "loss": 0.5304938316345215, + "mean_token_accuracy": 0.8463383808732032, + "num_tokens": 1306879.0, + "step": 560 + }, + { + "epoch": 1.3935242839352429, + "eval_entropy": 0.6099918867612995, + "eval_loss": 0.5620437860488892, + "eval_mean_token_accuracy": 0.8430728347495545, + "eval_num_tokens": 1306879.0, + "eval_runtime": 86.7741, + "eval_samples_per_second": 15.846, + "eval_steps_per_second": 1.982, + "step": 560 + }, + { + "entropy": 0.5768801040947438, + "epoch": 1.4433374844333748, + "grad_norm": 0.9198738932609558, + "learning_rate": 0.0002249458150352077, + "loss": 0.520513391494751, + "mean_token_accuracy": 0.8487689301371575, + "num_tokens": 1353534.0, + "step": 580 + }, + { + "epoch": 1.4433374844333748, + "eval_entropy": 0.6349420670506566, + "eval_loss": 0.5645340085029602, + "eval_mean_token_accuracy": 0.8447844597489335, + "eval_num_tokens": 1353534.0, + "eval_runtime": 86.3257, + "eval_samples_per_second": 15.928, + "eval_steps_per_second": 1.992, + "step": 580 + }, + { + "entropy": 0.5822233572602272, + "epoch": 1.4931506849315068, + "grad_norm": 0.832811176776886, + "learning_rate": 0.0002246281760679571, + "loss": 0.5295282363891601, + "mean_token_accuracy": 0.8504064798355102, + "num_tokens": 1399537.0, + "step": 600 + }, + { + "epoch": 1.4931506849315068, + "eval_entropy": 0.5829724387027496, + "eval_loss": 0.5612193942070007, + "eval_mean_token_accuracy": 0.8449643853791925, + "eval_num_tokens": 1399537.0, + "eval_runtime": 86.6617, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 600 + }, + { + "entropy": 0.571855777129531, + "epoch": 1.5429638854296388, + "grad_norm": 0.7665547728538513, + "learning_rate": 0.00022427691370553263, + "loss": 0.5187931060791016, + "mean_token_accuracy": 0.8534420043230057, + "num_tokens": 1448422.0, + "step": 620 + }, + { + "epoch": 1.5429638854296388, + "eval_entropy": 0.5623592240519302, + "eval_loss": 0.5575760006904602, + "eval_mean_token_accuracy": 0.8468210229346919, + "eval_num_tokens": 1448422.0, + "eval_runtime": 86.6324, + "eval_samples_per_second": 15.872, + "eval_steps_per_second": 1.985, + "step": 620 + }, + { + "entropy": 0.5740394659340382, + "epoch": 1.592777085927771, + "grad_norm": 0.6500429511070251, + "learning_rate": 0.00022389213388387174, + "loss": 0.5283198833465577, + "mean_token_accuracy": 0.8502798482775689, + "num_tokens": 1495009.0, + "step": 640 + }, + { + "epoch": 1.592777085927771, + "eval_entropy": 0.5548852207355721, + "eval_loss": 0.5561797022819519, + "eval_mean_token_accuracy": 0.8452786498291548, + "eval_num_tokens": 1495009.0, + "eval_runtime": 86.5205, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 640 + }, + { + "entropy": 0.6020145989954472, + "epoch": 1.6425902864259028, + "grad_norm": 0.7056867480278015, + "learning_rate": 0.00022347395264732053, + "loss": 0.5400049209594726, + "mean_token_accuracy": 0.8447613954544068, + "num_tokens": 1536932.0, + "step": 660 + }, + { + "epoch": 1.6425902864259028, + "eval_entropy": 0.5618055154417836, + "eval_loss": 0.556106686592102, + "eval_mean_token_accuracy": 0.8465680112672407, + "eval_num_tokens": 1536932.0, + "eval_runtime": 86.2971, + "eval_samples_per_second": 15.933, + "eval_steps_per_second": 1.993, + "step": 660 + }, + { + "entropy": 0.5665927153080702, + "epoch": 1.692403486924035, + "grad_norm": 0.5987663865089417, + "learning_rate": 0.00022302249611363625, + "loss": 0.5143643856048584, + "mean_token_accuracy": 0.8529589556157589, + "num_tokens": 1585718.0, + "step": 680 + }, + { + "epoch": 1.692403486924035, + "eval_entropy": 0.568248552118623, + "eval_loss": 0.5476346015930176, + "eval_mean_token_accuracy": 0.8476775434128073, + "eval_num_tokens": 1585718.0, + "eval_runtime": 86.9583, + "eval_samples_per_second": 15.812, + "eval_steps_per_second": 1.978, + "step": 680 + }, + { + "entropy": 0.5673687808215618, + "epoch": 1.7422166874221667, + "grad_norm": 0.735261857509613, + "learning_rate": 0.00022253790043595193, + "loss": 0.509885597229004, + "mean_token_accuracy": 0.8537046857178211, + "num_tokens": 1635718.0, + "step": 700 + }, + { + "epoch": 1.7422166874221667, + "eval_entropy": 0.5616967284748721, + "eval_loss": 0.5439274311065674, + "eval_mean_token_accuracy": 0.8488946217437123, + "eval_num_tokens": 1635718.0, + "eval_runtime": 86.0604, + "eval_samples_per_second": 15.977, + "eval_steps_per_second": 1.999, + "step": 700 + }, + { + "entropy": 0.5529541682451964, + "epoch": 1.792029887920299, + "grad_norm": 0.7014835476875305, + "learning_rate": 0.00022202031176171442, + "loss": 0.5078992366790771, + "mean_token_accuracy": 0.8525233261287213, + "num_tokens": 1681291.0, + "step": 720 + }, + { + "epoch": 1.792029887920299, + "eval_entropy": 0.5827173320359962, + "eval_loss": 0.5419450402259827, + "eval_mean_token_accuracy": 0.8477318609176681, + "eval_num_tokens": 1681291.0, + "eval_runtime": 85.2984, + "eval_samples_per_second": 16.12, + "eval_steps_per_second": 2.016, + "step": 720 + }, + { + "entropy": 0.5755720350891351, + "epoch": 1.841843088418431, + "grad_norm": 0.705613911151886, + "learning_rate": 0.00022146988618860824, + "loss": 0.5181350708007812, + "mean_token_accuracy": 0.8467609457671642, + "num_tokens": 1729102.0, + "step": 740 + }, + { + "epoch": 1.841843088418431, + "eval_entropy": 0.5743971356125765, + "eval_loss": 0.5415896773338318, + "eval_mean_token_accuracy": 0.847328585940738, + "eval_num_tokens": 1729102.0, + "eval_runtime": 85.5602, + "eval_samples_per_second": 16.071, + "eval_steps_per_second": 2.01, + "step": 740 + }, + { + "entropy": 0.561330484598875, + "epoch": 1.891656288916563, + "grad_norm": 0.6722865700721741, + "learning_rate": 0.0002208867897174789, + "loss": 0.499837589263916, + "mean_token_accuracy": 0.8518734864890576, + "num_tokens": 1773578.0, + "step": 760 + }, + { + "epoch": 1.891656288916563, + "eval_entropy": 0.5865232653396074, + "eval_loss": 0.5437926650047302, + "eval_mean_token_accuracy": 0.8450997017843779, + "eval_num_tokens": 1773578.0, + "eval_runtime": 86.4116, + "eval_samples_per_second": 15.912, + "eval_steps_per_second": 1.99, + "step": 760 + }, + { + "entropy": 0.547389242425561, + "epoch": 1.9414694894146949, + "grad_norm": 0.7935577034950256, + "learning_rate": 0.00022027119820226907, + "loss": 0.4977591514587402, + "mean_token_accuracy": 0.8539491161704064, + "num_tokens": 1821725.0, + "step": 780 + }, + { + "epoch": 1.9414694894146949, + "eval_entropy": 0.5290903090391048, + "eval_loss": 0.5409526824951172, + "eval_mean_token_accuracy": 0.8497545698354411, + "eval_num_tokens": 1821725.0, + "eval_runtime": 86.7262, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 780 + }, + { + "entropy": 0.5687909748405218, + "epoch": 1.9912826899128269, + "grad_norm": 0.6180546283721924, + "learning_rate": 0.00021962329729698345, + "loss": 0.5109643459320068, + "mean_token_accuracy": 0.8521598495543004, + "num_tokens": 1868431.0, + "step": 800 + }, + { + "epoch": 1.9912826899128269, + "eval_entropy": 0.5503541858390321, + "eval_loss": 0.5361555218696594, + "eval_mean_token_accuracy": 0.8510884285666221, + "eval_num_tokens": 1868431.0, + "eval_runtime": 86.3339, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 800 + }, + { + "entropy": 0.4739728841261986, + "epoch": 2.0398505603985058, + "grad_norm": 0.8058829307556152, + "learning_rate": 0.0002189432823996982, + "loss": 0.4204097747802734, + "mean_token_accuracy": 0.8728981889211215, + "num_tokens": 1915280.0, + "step": 820 + }, + { + "epoch": 2.0398505603985058, + "eval_entropy": 0.5077334992414297, + "eval_loss": 0.5531114339828491, + "eval_mean_token_accuracy": 0.8489257208136625, + "eval_num_tokens": 1915280.0, + "eval_runtime": 86.4801, + "eval_samples_per_second": 15.9, + "eval_steps_per_second": 1.989, + "step": 820 + }, + { + "entropy": 0.4594309840351343, + "epoch": 2.0896637608966375, + "grad_norm": 0.6906896829605103, + "learning_rate": 0.0002182313585936314, + "loss": 0.4071959495544434, + "mean_token_accuracy": 0.8732857562601566, + "num_tokens": 1965306.0, + "step": 840 + }, + { + "epoch": 2.0896637608966375, + "eval_entropy": 0.49850136994622474, + "eval_loss": 0.5486204624176025, + "eval_mean_token_accuracy": 0.8507991450470548, + "eval_num_tokens": 1965306.0, + "eval_runtime": 86.3364, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 840 + }, + { + "entropy": 0.4881629109382629, + "epoch": 2.1394769613947697, + "grad_norm": 0.6343470215797424, + "learning_rate": 0.0002174877405852928, + "loss": 0.41669540405273436, + "mean_token_accuracy": 0.8711295068264008, + "num_tokens": 2008562.0, + "step": 860 + }, + { + "epoch": 2.1394769613947697, + "eval_entropy": 0.49155513924914734, + "eval_loss": 0.555109441280365, + "eval_mean_token_accuracy": 0.8496399400539176, + "eval_num_tokens": 2008562.0, + "eval_runtime": 86.3295, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 860 + }, + { + "entropy": 0.4648668970912695, + "epoch": 2.1892901618929015, + "grad_norm": 0.8014165163040161, + "learning_rate": 0.00021671265263973133, + "loss": 0.4110250473022461, + "mean_token_accuracy": 0.8754166305065155, + "num_tokens": 2056474.0, + "step": 880 + }, + { + "epoch": 2.1892901618929015, + "eval_entropy": 0.4909258722219356, + "eval_loss": 0.5539511442184448, + "eval_mean_token_accuracy": 0.8492401502160138, + "eval_num_tokens": 2056474.0, + "eval_runtime": 86.3468, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 880 + }, + { + "entropy": 0.4824485514312983, + "epoch": 2.2391033623910337, + "grad_norm": 0.6665191054344177, + "learning_rate": 0.00021590632851289967, + "loss": 0.4181404113769531, + "mean_token_accuracy": 0.8726993151009083, + "num_tokens": 2103543.0, + "step": 900 + }, + { + "epoch": 2.2391033623910337, + "eval_entropy": 0.4986876940657926, + "eval_loss": 0.547695517539978, + "eval_mean_token_accuracy": 0.8501384708770486, + "eval_num_tokens": 2103543.0, + "eval_runtime": 86.3838, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 900 + }, + { + "entropy": 0.4751896943897009, + "epoch": 2.2889165628891655, + "grad_norm": 0.81158047914505, + "learning_rate": 0.00021506901138115678, + "loss": 0.40689678192138673, + "mean_token_accuracy": 0.8745221219956875, + "num_tokens": 2147861.0, + "step": 920 + }, + { + "epoch": 2.2889165628891655, + "eval_entropy": 0.507153491121392, + "eval_loss": 0.5501641631126404, + "eval_mean_token_accuracy": 0.8495670116918032, + "eval_num_tokens": 2147861.0, + "eval_runtime": 86.0912, + "eval_samples_per_second": 15.971, + "eval_steps_per_second": 1.998, + "step": 920 + }, + { + "entropy": 0.4873133715242147, + "epoch": 2.3387297633872977, + "grad_norm": 0.7218056321144104, + "learning_rate": 0.0002142009537679292, + "loss": 0.42701358795166017, + "mean_token_accuracy": 0.8695114746689796, + "num_tokens": 2190561.0, + "step": 940 + }, + { + "epoch": 2.3387297633872977, + "eval_entropy": 0.5202612736543943, + "eval_loss": 0.5491839051246643, + "eval_mean_token_accuracy": 0.8494071208460386, + "eval_num_tokens": 2190561.0, + "eval_runtime": 86.1142, + "eval_samples_per_second": 15.967, + "eval_steps_per_second": 1.997, + "step": 940 + }, + { + "entropy": 0.4762951169162989, + "epoch": 2.3885429638854294, + "grad_norm": 0.7194424867630005, + "learning_rate": 0.0002133024174675534, + "loss": 0.42299847602844237, + "mean_token_accuracy": 0.8709790132939815, + "num_tokens": 2239412.0, + "step": 960 + }, + { + "epoch": 2.3885429638854294, + "eval_entropy": 0.4899340462546016, + "eval_loss": 0.5522511601448059, + "eval_mean_token_accuracy": 0.8492208258357159, + "eval_num_tokens": 2239412.0, + "eval_runtime": 86.463, + "eval_samples_per_second": 15.903, + "eval_steps_per_second": 1.989, + "step": 960 + }, + { + "entropy": 0.49650347977876663, + "epoch": 2.4383561643835616, + "grad_norm": 0.8406022787094116, + "learning_rate": 0.0002123736734663221, + "loss": 0.4275330066680908, + "mean_token_accuracy": 0.8670595556497573, + "num_tokens": 2286283.0, + "step": 980 + }, + { + "epoch": 2.4383561643835616, + "eval_entropy": 0.49691385654515996, + "eval_loss": 0.5491269826889038, + "eval_mean_token_accuracy": 0.850309816210769, + "eval_num_tokens": 2286283.0, + "eval_runtime": 86.17, + "eval_samples_per_second": 15.957, + "eval_steps_per_second": 1.996, + "step": 980 + }, + { + "entropy": 0.48843890577554705, + "epoch": 2.488169364881694, + "grad_norm": 0.9082473516464233, + "learning_rate": 0.00021141500186075868, + "loss": 0.4309722423553467, + "mean_token_accuracy": 0.8686766296625137, + "num_tokens": 2333733.0, + "step": 1000 + }, + { + "epoch": 2.488169364881694, + "eval_entropy": 0.5543508351195691, + "eval_loss": 0.5478800535202026, + "eval_mean_token_accuracy": 0.8478029522784921, + "eval_num_tokens": 2333733.0, + "eval_runtime": 86.3835, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 1000 + }, + { + "entropy": 0.4777219031006098, + "epoch": 2.5379825653798256, + "grad_norm": 0.7448089122772217, + "learning_rate": 0.0002104266917731438, + "loss": 0.423325252532959, + "mean_token_accuracy": 0.8706337086856365, + "num_tokens": 2384270.0, + "step": 1020 + }, + { + "epoch": 2.5379825653798256, + "eval_entropy": 0.49857561550168106, + "eval_loss": 0.5511948466300964, + "eval_mean_token_accuracy": 0.8502220289651737, + "eval_num_tokens": 2384270.0, + "eval_runtime": 86.5399, + "eval_samples_per_second": 15.889, + "eval_steps_per_second": 1.988, + "step": 1020 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0076952699436032e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c1d5938937c6a68e305951054e9038c0386b165a --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1040/trainer_state.json @@ -0,0 +1,1126 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.587795765877958, + "eval_steps": 20, + "global_step": 1040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + }, + { + "entropy": 0.6744543805718421, + "epoch": 0.298879202988792, + "grad_norm": 0.932099461555481, + "learning_rate": 6.698322232264434e-05, + "loss": 0.5991750717163086, + "mean_token_accuracy": 0.8304223112761975, + "num_tokens": 281241.0, + "step": 120 + }, + { + "epoch": 0.298879202988792, + "eval_entropy": 0.6813044282932614, + "eval_loss": 0.5922021269798279, + "eval_mean_token_accuracy": 0.8346439617317777, + "eval_num_tokens": 281241.0, + "eval_runtime": 86.1551, + "eval_samples_per_second": 15.96, + "eval_steps_per_second": 1.996, + "step": 120 + }, + { + "entropy": 0.6663189359009266, + "epoch": 0.34869240348692404, + "grad_norm": 0.9528499841690063, + "learning_rate": 7.824090674661818e-05, + "loss": 0.5891091346740722, + "mean_token_accuracy": 0.832152470946312, + "num_tokens": 327393.0, + "step": 140 + }, + { + "epoch": 0.34869240348692404, + "eval_entropy": 0.6398407208711602, + "eval_loss": 0.5859636664390564, + "eval_mean_token_accuracy": 0.8372074996316156, + "eval_num_tokens": 327393.0, + "eval_runtime": 86.2706, + "eval_samples_per_second": 15.938, + "eval_steps_per_second": 1.994, + "step": 140 + }, + { + "entropy": 0.64859763905406, + "epoch": 0.398505603985056, + "grad_norm": 0.8468204140663147, + "learning_rate": 8.949859117059201e-05, + "loss": 0.569426441192627, + "mean_token_accuracy": 0.8401990942656994, + "num_tokens": 373834.0, + "step": 160 + }, + { + "epoch": 0.398505603985056, + "eval_entropy": 0.6381674285891444, + "eval_loss": 0.5744525790214539, + "eval_mean_token_accuracy": 0.838626817908398, + "eval_num_tokens": 373834.0, + "eval_runtime": 86.2848, + "eval_samples_per_second": 15.936, + "eval_steps_per_second": 1.993, + "step": 160 + }, + { + "entropy": 0.6432608783245086, + "epoch": 0.44831880448318806, + "grad_norm": 0.8765804767608643, + "learning_rate": 0.00010075627559456587, + "loss": 0.5687318801879883, + "mean_token_accuracy": 0.839249350130558, + "num_tokens": 422572.0, + "step": 180 + }, + { + "epoch": 0.44831880448318806, + "eval_entropy": 0.6047098288355872, + "eval_loss": 0.5679298043251038, + "eval_mean_token_accuracy": 0.8410577181466791, + "eval_num_tokens": 422572.0, + "eval_runtime": 86.5879, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.986, + "step": 180 + }, + { + "entropy": 0.6440276011824608, + "epoch": 0.49813200498132004, + "grad_norm": 0.9576020240783691, + "learning_rate": 0.00011201396001853971, + "loss": 0.5828506469726562, + "mean_token_accuracy": 0.837553184479475, + "num_tokens": 471879.0, + "step": 200 + }, + { + "epoch": 0.49813200498132004, + "eval_entropy": 0.6161119360909906, + "eval_loss": 0.5702911615371704, + "eval_mean_token_accuracy": 0.8407089398350827, + "eval_num_tokens": 471879.0, + "eval_runtime": 86.3341, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 200 + }, + { + "entropy": 0.6377195850014686, + "epoch": 0.547945205479452, + "grad_norm": 0.7212373614311218, + "learning_rate": 0.00012327164444251353, + "loss": 0.5702451229095459, + "mean_token_accuracy": 0.8397969007492065, + "num_tokens": 520984.0, + "step": 220 + }, + { + "epoch": 0.547945205479452, + "eval_entropy": 0.6080108886194784, + "eval_loss": 0.5633499622344971, + "eval_mean_token_accuracy": 0.8396634854549585, + "eval_num_tokens": 520984.0, + "eval_runtime": 86.4945, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 220 + }, + { + "entropy": 0.6287345830351114, + "epoch": 0.597758405977584, + "grad_norm": 0.848779022693634, + "learning_rate": 0.00013452932886648739, + "loss": 0.5506546020507812, + "mean_token_accuracy": 0.8438881888985634, + "num_tokens": 566596.0, + "step": 240 + }, + { + "epoch": 0.597758405977584, + "eval_entropy": 0.6307531505130058, + "eval_loss": 0.5573338270187378, + "eval_mean_token_accuracy": 0.8431362606758295, + "eval_num_tokens": 566596.0, + "eval_runtime": 86.3535, + "eval_samples_per_second": 15.923, + "eval_steps_per_second": 1.992, + "step": 240 + }, + { + "entropy": 0.6223786748945713, + "epoch": 0.6475716064757161, + "grad_norm": 0.7316951751708984, + "learning_rate": 0.0001457870132904612, + "loss": 0.5495625972747803, + "mean_token_accuracy": 0.8440376669168472, + "num_tokens": 613603.0, + "step": 260 + }, + { + "epoch": 0.6475716064757161, + "eval_entropy": 0.623454462476941, + "eval_loss": 0.5619264245033264, + "eval_mean_token_accuracy": 0.8431175777385401, + "eval_num_tokens": 613603.0, + "eval_runtime": 86.2008, + "eval_samples_per_second": 15.951, + "eval_steps_per_second": 1.995, + "step": 260 + }, + { + "entropy": 0.6281675305217505, + "epoch": 0.6973848069738481, + "grad_norm": 0.7639564871788025, + "learning_rate": 0.00015704469771443506, + "loss": 0.5604369163513183, + "mean_token_accuracy": 0.8401600055396556, + "num_tokens": 658565.0, + "step": 280 + }, + { + "epoch": 0.6973848069738481, + "eval_entropy": 0.63416675980701, + "eval_loss": 0.5612760782241821, + "eval_mean_token_accuracy": 0.842435666294985, + "eval_num_tokens": 658565.0, + "eval_runtime": 86.25, + "eval_samples_per_second": 15.942, + "eval_steps_per_second": 1.994, + "step": 280 + }, + { + "entropy": 0.6427909277379513, + "epoch": 0.7471980074719801, + "grad_norm": 0.6475813388824463, + "learning_rate": 0.0001683023821384089, + "loss": 0.573763370513916, + "mean_token_accuracy": 0.8370340794324875, + "num_tokens": 705680.0, + "step": 300 + }, + { + "epoch": 0.7471980074719801, + "eval_entropy": 0.6231539840268534, + "eval_loss": 0.5566866397857666, + "eval_mean_token_accuracy": 0.844177934319474, + "eval_num_tokens": 705680.0, + "eval_runtime": 86.4858, + "eval_samples_per_second": 15.899, + "eval_steps_per_second": 1.989, + "step": 300 + }, + { + "entropy": 0.6226776849478484, + "epoch": 0.797011207970112, + "grad_norm": 0.8886699676513672, + "learning_rate": 0.00017956006656238274, + "loss": 0.558210802078247, + "mean_token_accuracy": 0.84083157107234, + "num_tokens": 752616.0, + "step": 320 + }, + { + "epoch": 0.797011207970112, + "eval_entropy": 0.6066981683983359, + "eval_loss": 0.5585207939147949, + "eval_mean_token_accuracy": 0.8423153311014175, + "eval_num_tokens": 752616.0, + "eval_runtime": 86.3463, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 320 + }, + { + "entropy": 0.6249004438519478, + "epoch": 0.8468244084682441, + "grad_norm": 0.8791211843490601, + "learning_rate": 0.00019081775098635657, + "loss": 0.5603597164154053, + "mean_token_accuracy": 0.8420463085174561, + "num_tokens": 797151.0, + "step": 340 + }, + { + "epoch": 0.8468244084682441, + "eval_entropy": 0.6082247584018596, + "eval_loss": 0.5616299510002136, + "eval_mean_token_accuracy": 0.8431286801432454, + "eval_num_tokens": 797151.0, + "eval_runtime": 86.1253, + "eval_samples_per_second": 15.965, + "eval_steps_per_second": 1.997, + "step": 340 + }, + { + "entropy": 0.6362396612763405, + "epoch": 0.8966376089663761, + "grad_norm": 0.8606319427490234, + "learning_rate": 0.0002020754354103304, + "loss": 0.5735773563385009, + "mean_token_accuracy": 0.8371490836143494, + "num_tokens": 843585.0, + "step": 360 + }, + { + "epoch": 0.8966376089663761, + "eval_entropy": 0.6492362072648004, + "eval_loss": 0.5646467804908752, + "eval_mean_token_accuracy": 0.8415517574825953, + "eval_num_tokens": 843585.0, + "eval_runtime": 86.3351, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 360 + }, + { + "entropy": 0.638665035739541, + "epoch": 0.9464508094645081, + "grad_norm": 0.7773950099945068, + "learning_rate": 0.00021333311983430425, + "loss": 0.5820859909057617, + "mean_token_accuracy": 0.8372561208903789, + "num_tokens": 889842.0, + "step": 380 + }, + { + "epoch": 0.9464508094645081, + "eval_entropy": 0.6434498637221581, + "eval_loss": 0.5645168423652649, + "eval_mean_token_accuracy": 0.8420382481674815, + "eval_num_tokens": 889842.0, + "eval_runtime": 86.1216, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 380 + }, + { + "entropy": 0.6316851265728474, + "epoch": 0.9962640099626401, + "grad_norm": 1.6120579242706299, + "learning_rate": 0.00022459080425827807, + "loss": 0.5637502670288086, + "mean_token_accuracy": 0.8386227294802666, + "num_tokens": 935589.0, + "step": 400 + }, + { + "epoch": 0.9962640099626401, + "eval_entropy": 0.6469012776086497, + "eval_loss": 0.5758090615272522, + "eval_mean_token_accuracy": 0.8397158470957778, + "eval_num_tokens": 935589.0, + "eval_runtime": 86.6139, + "eval_samples_per_second": 15.875, + "eval_steps_per_second": 1.986, + "step": 400 + }, + { + "entropy": 0.5894816922835815, + "epoch": 1.0448318804483188, + "grad_norm": 1.1616325378417969, + "learning_rate": 0.00022626713048053178, + "loss": 0.5316025257110596, + "mean_token_accuracy": 0.8466163017810919, + "num_tokens": 980589.0, + "step": 420 + }, + { + "epoch": 1.0448318804483188, + "eval_entropy": 0.5860798164855602, + "eval_loss": 0.5777581930160522, + "eval_mean_token_accuracy": 0.8396938103576039, + "eval_num_tokens": 980589.0, + "eval_runtime": 86.1449, + "eval_samples_per_second": 15.961, + "eval_steps_per_second": 1.997, + "step": 420 + }, + { + "entropy": 0.5818420693278312, + "epoch": 1.0946450809464507, + "grad_norm": 0.7999453544616699, + "learning_rate": 0.00022622107023288778, + "loss": 0.5221010208129883, + "mean_token_accuracy": 0.8474301159381866, + "num_tokens": 1027852.0, + "step": 440 + }, + { + "epoch": 1.0946450809464507, + "eval_entropy": 0.5783926014636838, + "eval_loss": 0.5700300931930542, + "eval_mean_token_accuracy": 0.8430753537388735, + "eval_num_tokens": 1027852.0, + "eval_runtime": 86.5308, + "eval_samples_per_second": 15.89, + "eval_steps_per_second": 1.988, + "step": 440 + }, + { + "entropy": 0.5612493887543678, + "epoch": 1.1444582814445827, + "grad_norm": 1.015687346458435, + "learning_rate": 0.00022614090619491568, + "loss": 0.5084867000579834, + "mean_token_accuracy": 0.8495561093091964, + "num_tokens": 1077649.0, + "step": 460 + }, + { + "epoch": 1.1444582814445827, + "eval_entropy": 0.5841563874205877, + "eval_loss": 0.5693665742874146, + "eval_mean_token_accuracy": 0.8427817298229351, + "eval_num_tokens": 1077649.0, + "eval_runtime": 86.5256, + "eval_samples_per_second": 15.891, + "eval_steps_per_second": 1.988, + "step": 460 + }, + { + "entropy": 0.5828216474503278, + "epoch": 1.1942714819427147, + "grad_norm": 1.9750930070877075, + "learning_rate": 0.00022602666254299594, + "loss": 0.5180017948150635, + "mean_token_accuracy": 0.8515685826539994, + "num_tokens": 1124872.0, + "step": 480 + }, + { + "epoch": 1.1942714819427147, + "eval_entropy": 0.5806607044366903, + "eval_loss": 0.5804352760314941, + "eval_mean_token_accuracy": 0.8413014668364858, + "eval_num_tokens": 1124872.0, + "eval_runtime": 86.1199, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 480 + }, + { + "entropy": 0.5926914308220148, + "epoch": 1.244084682440847, + "grad_norm": 0.8917353749275208, + "learning_rate": 0.0002258783737314558, + "loss": 0.528910779953003, + "mean_token_accuracy": 0.8486074328422546, + "num_tokens": 1168698.0, + "step": 500 + }, + { + "epoch": 1.244084682440847, + "eval_entropy": 0.5593361884009006, + "eval_loss": 0.5675153732299805, + "eval_mean_token_accuracy": 0.8433507802181466, + "eval_num_tokens": 1168698.0, + "eval_runtime": 86.7289, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 500 + }, + { + "entropy": 0.5865630559623242, + "epoch": 1.293897882938979, + "grad_norm": 0.7482362985610962, + "learning_rate": 0.00022569608448217823, + "loss": 0.5250466823577881, + "mean_token_accuracy": 0.8477916084229946, + "num_tokens": 1216679.0, + "step": 520 + }, + { + "epoch": 1.293897882938979, + "eval_entropy": 0.543057840230853, + "eval_loss": 0.5671008229255676, + "eval_mean_token_accuracy": 0.8428726016088973, + "eval_num_tokens": 1216679.0, + "eval_runtime": 86.3403, + "eval_samples_per_second": 15.925, + "eval_steps_per_second": 1.992, + "step": 520 + }, + { + "entropy": 0.5870206747204065, + "epoch": 1.3437110834371109, + "grad_norm": 0.9473814964294434, + "learning_rate": 0.00022547984977111448, + "loss": 0.5252370834350586, + "mean_token_accuracy": 0.8468369916081429, + "num_tokens": 1261365.0, + "step": 540 + }, + { + "epoch": 1.3437110834371109, + "eval_entropy": 0.590982622878496, + "eval_loss": 0.5676343441009521, + "eval_mean_token_accuracy": 0.8429348746011424, + "eval_num_tokens": 1261365.0, + "eval_runtime": 86.5168, + "eval_samples_per_second": 15.893, + "eval_steps_per_second": 1.988, + "step": 540 + }, + { + "entropy": 0.5785854265093804, + "epoch": 1.3935242839352429, + "grad_norm": 0.9353351593017578, + "learning_rate": 0.0002252297348117042, + "loss": 0.5304938316345215, + "mean_token_accuracy": 0.8463383808732032, + "num_tokens": 1306879.0, + "step": 560 + }, + { + "epoch": 1.3935242839352429, + "eval_entropy": 0.6099918867612995, + "eval_loss": 0.5620437860488892, + "eval_mean_token_accuracy": 0.8430728347495545, + "eval_num_tokens": 1306879.0, + "eval_runtime": 86.7741, + "eval_samples_per_second": 15.846, + "eval_steps_per_second": 1.982, + "step": 560 + }, + { + "entropy": 0.5768801040947438, + "epoch": 1.4433374844333748, + "grad_norm": 0.9198738932609558, + "learning_rate": 0.0002249458150352077, + "loss": 0.520513391494751, + "mean_token_accuracy": 0.8487689301371575, + "num_tokens": 1353534.0, + "step": 580 + }, + { + "epoch": 1.4433374844333748, + "eval_entropy": 0.6349420670506566, + "eval_loss": 0.5645340085029602, + "eval_mean_token_accuracy": 0.8447844597489335, + "eval_num_tokens": 1353534.0, + "eval_runtime": 86.3257, + "eval_samples_per_second": 15.928, + "eval_steps_per_second": 1.992, + "step": 580 + }, + { + "entropy": 0.5822233572602272, + "epoch": 1.4931506849315068, + "grad_norm": 0.832811176776886, + "learning_rate": 0.0002246281760679571, + "loss": 0.5295282363891601, + "mean_token_accuracy": 0.8504064798355102, + "num_tokens": 1399537.0, + "step": 600 + }, + { + "epoch": 1.4931506849315068, + "eval_entropy": 0.5829724387027496, + "eval_loss": 0.5612193942070007, + "eval_mean_token_accuracy": 0.8449643853791925, + "eval_num_tokens": 1399537.0, + "eval_runtime": 86.6617, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 600 + }, + { + "entropy": 0.571855777129531, + "epoch": 1.5429638854296388, + "grad_norm": 0.7665547728538513, + "learning_rate": 0.00022427691370553263, + "loss": 0.5187931060791016, + "mean_token_accuracy": 0.8534420043230057, + "num_tokens": 1448422.0, + "step": 620 + }, + { + "epoch": 1.5429638854296388, + "eval_entropy": 0.5623592240519302, + "eval_loss": 0.5575760006904602, + "eval_mean_token_accuracy": 0.8468210229346919, + "eval_num_tokens": 1448422.0, + "eval_runtime": 86.6324, + "eval_samples_per_second": 15.872, + "eval_steps_per_second": 1.985, + "step": 620 + }, + { + "entropy": 0.5740394659340382, + "epoch": 1.592777085927771, + "grad_norm": 0.6500429511070251, + "learning_rate": 0.00022389213388387174, + "loss": 0.5283198833465577, + "mean_token_accuracy": 0.8502798482775689, + "num_tokens": 1495009.0, + "step": 640 + }, + { + "epoch": 1.592777085927771, + "eval_entropy": 0.5548852207355721, + "eval_loss": 0.5561797022819519, + "eval_mean_token_accuracy": 0.8452786498291548, + "eval_num_tokens": 1495009.0, + "eval_runtime": 86.5205, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 640 + }, + { + "entropy": 0.6020145989954472, + "epoch": 1.6425902864259028, + "grad_norm": 0.7056867480278015, + "learning_rate": 0.00022347395264732053, + "loss": 0.5400049209594726, + "mean_token_accuracy": 0.8447613954544068, + "num_tokens": 1536932.0, + "step": 660 + }, + { + "epoch": 1.6425902864259028, + "eval_entropy": 0.5618055154417836, + "eval_loss": 0.556106686592102, + "eval_mean_token_accuracy": 0.8465680112672407, + "eval_num_tokens": 1536932.0, + "eval_runtime": 86.2971, + "eval_samples_per_second": 15.933, + "eval_steps_per_second": 1.993, + "step": 660 + }, + { + "entropy": 0.5665927153080702, + "epoch": 1.692403486924035, + "grad_norm": 0.5987663865089417, + "learning_rate": 0.00022302249611363625, + "loss": 0.5143643856048584, + "mean_token_accuracy": 0.8529589556157589, + "num_tokens": 1585718.0, + "step": 680 + }, + { + "epoch": 1.692403486924035, + "eval_entropy": 0.568248552118623, + "eval_loss": 0.5476346015930176, + "eval_mean_token_accuracy": 0.8476775434128073, + "eval_num_tokens": 1585718.0, + "eval_runtime": 86.9583, + "eval_samples_per_second": 15.812, + "eval_steps_per_second": 1.978, + "step": 680 + }, + { + "entropy": 0.5673687808215618, + "epoch": 1.7422166874221667, + "grad_norm": 0.735261857509613, + "learning_rate": 0.00022253790043595193, + "loss": 0.509885597229004, + "mean_token_accuracy": 0.8537046857178211, + "num_tokens": 1635718.0, + "step": 700 + }, + { + "epoch": 1.7422166874221667, + "eval_entropy": 0.5616967284748721, + "eval_loss": 0.5439274311065674, + "eval_mean_token_accuracy": 0.8488946217437123, + "eval_num_tokens": 1635718.0, + "eval_runtime": 86.0604, + "eval_samples_per_second": 15.977, + "eval_steps_per_second": 1.999, + "step": 700 + }, + { + "entropy": 0.5529541682451964, + "epoch": 1.792029887920299, + "grad_norm": 0.7014835476875305, + "learning_rate": 0.00022202031176171442, + "loss": 0.5078992366790771, + "mean_token_accuracy": 0.8525233261287213, + "num_tokens": 1681291.0, + "step": 720 + }, + { + "epoch": 1.792029887920299, + "eval_entropy": 0.5827173320359962, + "eval_loss": 0.5419450402259827, + "eval_mean_token_accuracy": 0.8477318609176681, + "eval_num_tokens": 1681291.0, + "eval_runtime": 85.2984, + "eval_samples_per_second": 16.12, + "eval_steps_per_second": 2.016, + "step": 720 + }, + { + "entropy": 0.5755720350891351, + "epoch": 1.841843088418431, + "grad_norm": 0.705613911151886, + "learning_rate": 0.00022146988618860824, + "loss": 0.5181350708007812, + "mean_token_accuracy": 0.8467609457671642, + "num_tokens": 1729102.0, + "step": 740 + }, + { + "epoch": 1.841843088418431, + "eval_entropy": 0.5743971356125765, + "eval_loss": 0.5415896773338318, + "eval_mean_token_accuracy": 0.847328585940738, + "eval_num_tokens": 1729102.0, + "eval_runtime": 85.5602, + "eval_samples_per_second": 16.071, + "eval_steps_per_second": 2.01, + "step": 740 + }, + { + "entropy": 0.561330484598875, + "epoch": 1.891656288916563, + "grad_norm": 0.6722865700721741, + "learning_rate": 0.0002208867897174789, + "loss": 0.499837589263916, + "mean_token_accuracy": 0.8518734864890576, + "num_tokens": 1773578.0, + "step": 760 + }, + { + "epoch": 1.891656288916563, + "eval_entropy": 0.5865232653396074, + "eval_loss": 0.5437926650047302, + "eval_mean_token_accuracy": 0.8450997017843779, + "eval_num_tokens": 1773578.0, + "eval_runtime": 86.4116, + "eval_samples_per_second": 15.912, + "eval_steps_per_second": 1.99, + "step": 760 + }, + { + "entropy": 0.547389242425561, + "epoch": 1.9414694894146949, + "grad_norm": 0.7935577034950256, + "learning_rate": 0.00022027119820226907, + "loss": 0.4977591514587402, + "mean_token_accuracy": 0.8539491161704064, + "num_tokens": 1821725.0, + "step": 780 + }, + { + "epoch": 1.9414694894146949, + "eval_entropy": 0.5290903090391048, + "eval_loss": 0.5409526824951172, + "eval_mean_token_accuracy": 0.8497545698354411, + "eval_num_tokens": 1821725.0, + "eval_runtime": 86.7262, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 780 + }, + { + "entropy": 0.5687909748405218, + "epoch": 1.9912826899128269, + "grad_norm": 0.6180546283721924, + "learning_rate": 0.00021962329729698345, + "loss": 0.5109643459320068, + "mean_token_accuracy": 0.8521598495543004, + "num_tokens": 1868431.0, + "step": 800 + }, + { + "epoch": 1.9912826899128269, + "eval_entropy": 0.5503541858390321, + "eval_loss": 0.5361555218696594, + "eval_mean_token_accuracy": 0.8510884285666221, + "eval_num_tokens": 1868431.0, + "eval_runtime": 86.3339, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 800 + }, + { + "entropy": 0.4739728841261986, + "epoch": 2.0398505603985058, + "grad_norm": 0.8058829307556152, + "learning_rate": 0.0002189432823996982, + "loss": 0.4204097747802734, + "mean_token_accuracy": 0.8728981889211215, + "num_tokens": 1915280.0, + "step": 820 + }, + { + "epoch": 2.0398505603985058, + "eval_entropy": 0.5077334992414297, + "eval_loss": 0.5531114339828491, + "eval_mean_token_accuracy": 0.8489257208136625, + "eval_num_tokens": 1915280.0, + "eval_runtime": 86.4801, + "eval_samples_per_second": 15.9, + "eval_steps_per_second": 1.989, + "step": 820 + }, + { + "entropy": 0.4594309840351343, + "epoch": 2.0896637608966375, + "grad_norm": 0.6906896829605103, + "learning_rate": 0.0002182313585936314, + "loss": 0.4071959495544434, + "mean_token_accuracy": 0.8732857562601566, + "num_tokens": 1965306.0, + "step": 840 + }, + { + "epoch": 2.0896637608966375, + "eval_entropy": 0.49850136994622474, + "eval_loss": 0.5486204624176025, + "eval_mean_token_accuracy": 0.8507991450470548, + "eval_num_tokens": 1965306.0, + "eval_runtime": 86.3364, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 840 + }, + { + "entropy": 0.4881629109382629, + "epoch": 2.1394769613947697, + "grad_norm": 0.6343470215797424, + "learning_rate": 0.0002174877405852928, + "loss": 0.41669540405273436, + "mean_token_accuracy": 0.8711295068264008, + "num_tokens": 2008562.0, + "step": 860 + }, + { + "epoch": 2.1394769613947697, + "eval_entropy": 0.49155513924914734, + "eval_loss": 0.555109441280365, + "eval_mean_token_accuracy": 0.8496399400539176, + "eval_num_tokens": 2008562.0, + "eval_runtime": 86.3295, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 860 + }, + { + "entropy": 0.4648668970912695, + "epoch": 2.1892901618929015, + "grad_norm": 0.8014165163040161, + "learning_rate": 0.00021671265263973133, + "loss": 0.4110250473022461, + "mean_token_accuracy": 0.8754166305065155, + "num_tokens": 2056474.0, + "step": 880 + }, + { + "epoch": 2.1892901618929015, + "eval_entropy": 0.4909258722219356, + "eval_loss": 0.5539511442184448, + "eval_mean_token_accuracy": 0.8492401502160138, + "eval_num_tokens": 2056474.0, + "eval_runtime": 86.3468, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 880 + }, + { + "entropy": 0.4824485514312983, + "epoch": 2.2391033623910337, + "grad_norm": 0.6665191054344177, + "learning_rate": 0.00021590632851289967, + "loss": 0.4181404113769531, + "mean_token_accuracy": 0.8726993151009083, + "num_tokens": 2103543.0, + "step": 900 + }, + { + "epoch": 2.2391033623910337, + "eval_entropy": 0.4986876940657926, + "eval_loss": 0.547695517539978, + "eval_mean_token_accuracy": 0.8501384708770486, + "eval_num_tokens": 2103543.0, + "eval_runtime": 86.3838, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 900 + }, + { + "entropy": 0.4751896943897009, + "epoch": 2.2889165628891655, + "grad_norm": 0.81158047914505, + "learning_rate": 0.00021506901138115678, + "loss": 0.40689678192138673, + "mean_token_accuracy": 0.8745221219956875, + "num_tokens": 2147861.0, + "step": 920 + }, + { + "epoch": 2.2889165628891655, + "eval_entropy": 0.507153491121392, + "eval_loss": 0.5501641631126404, + "eval_mean_token_accuracy": 0.8495670116918032, + "eval_num_tokens": 2147861.0, + "eval_runtime": 86.0912, + "eval_samples_per_second": 15.971, + "eval_steps_per_second": 1.998, + "step": 920 + }, + { + "entropy": 0.4873133715242147, + "epoch": 2.3387297633872977, + "grad_norm": 0.7218056321144104, + "learning_rate": 0.0002142009537679292, + "loss": 0.42701358795166017, + "mean_token_accuracy": 0.8695114746689796, + "num_tokens": 2190561.0, + "step": 940 + }, + { + "epoch": 2.3387297633872977, + "eval_entropy": 0.5202612736543943, + "eval_loss": 0.5491839051246643, + "eval_mean_token_accuracy": 0.8494071208460386, + "eval_num_tokens": 2190561.0, + "eval_runtime": 86.1142, + "eval_samples_per_second": 15.967, + "eval_steps_per_second": 1.997, + "step": 940 + }, + { + "entropy": 0.4762951169162989, + "epoch": 2.3885429638854294, + "grad_norm": 0.7194424867630005, + "learning_rate": 0.0002133024174675534, + "loss": 0.42299847602844237, + "mean_token_accuracy": 0.8709790132939815, + "num_tokens": 2239412.0, + "step": 960 + }, + { + "epoch": 2.3885429638854294, + "eval_entropy": 0.4899340462546016, + "eval_loss": 0.5522511601448059, + "eval_mean_token_accuracy": 0.8492208258357159, + "eval_num_tokens": 2239412.0, + "eval_runtime": 86.463, + "eval_samples_per_second": 15.903, + "eval_steps_per_second": 1.989, + "step": 960 + }, + { + "entropy": 0.49650347977876663, + "epoch": 2.4383561643835616, + "grad_norm": 0.8406022787094116, + "learning_rate": 0.0002123736734663221, + "loss": 0.4275330066680908, + "mean_token_accuracy": 0.8670595556497573, + "num_tokens": 2286283.0, + "step": 980 + }, + { + "epoch": 2.4383561643835616, + "eval_entropy": 0.49691385654515996, + "eval_loss": 0.5491269826889038, + "eval_mean_token_accuracy": 0.850309816210769, + "eval_num_tokens": 2286283.0, + "eval_runtime": 86.17, + "eval_samples_per_second": 15.957, + "eval_steps_per_second": 1.996, + "step": 980 + }, + { + "entropy": 0.48843890577554705, + "epoch": 2.488169364881694, + "grad_norm": 0.9082473516464233, + "learning_rate": 0.00021141500186075868, + "loss": 0.4309722423553467, + "mean_token_accuracy": 0.8686766296625137, + "num_tokens": 2333733.0, + "step": 1000 + }, + { + "epoch": 2.488169364881694, + "eval_entropy": 0.5543508351195691, + "eval_loss": 0.5478800535202026, + "eval_mean_token_accuracy": 0.8478029522784921, + "eval_num_tokens": 2333733.0, + "eval_runtime": 86.3835, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 1000 + }, + { + "entropy": 0.4777219031006098, + "epoch": 2.5379825653798256, + "grad_norm": 0.7448089122772217, + "learning_rate": 0.0002104266917731438, + "loss": 0.423325252532959, + "mean_token_accuracy": 0.8706337086856365, + "num_tokens": 2384270.0, + "step": 1020 + }, + { + "epoch": 2.5379825653798256, + "eval_entropy": 0.49857561550168106, + "eval_loss": 0.5511948466300964, + "eval_mean_token_accuracy": 0.8502220289651737, + "eval_num_tokens": 2384270.0, + "eval_runtime": 86.5399, + "eval_samples_per_second": 15.889, + "eval_steps_per_second": 1.988, + "step": 1020 + }, + { + "entropy": 0.4844174191355705, + "epoch": 2.587795765877958, + "grad_norm": 0.794029176235199, + "learning_rate": 0.00020940904126432, + "loss": 0.4176753044128418, + "mean_token_accuracy": 0.873535567522049, + "num_tokens": 2428036.0, + "step": 1040 + }, + { + "epoch": 2.587795765877958, + "eval_entropy": 0.485467542222766, + "eval_loss": 0.5539286732673645, + "eval_mean_token_accuracy": 0.8495475081510322, + "eval_num_tokens": 2428036.0, + "eval_runtime": 86.135, + "eval_samples_per_second": 15.963, + "eval_steps_per_second": 1.997, + "step": 1040 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0254458345271091e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..506fa4843af506db69bfb8e8c5f6eee7a56cad8a --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1060/trainer_state.json @@ -0,0 +1,1147 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.6376089663760895, + "eval_steps": 20, + "global_step": 1060, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + }, + { + "entropy": 0.6744543805718421, + "epoch": 0.298879202988792, + "grad_norm": 0.932099461555481, + "learning_rate": 6.698322232264434e-05, + "loss": 0.5991750717163086, + "mean_token_accuracy": 0.8304223112761975, + "num_tokens": 281241.0, + "step": 120 + }, + { + "epoch": 0.298879202988792, + "eval_entropy": 0.6813044282932614, + "eval_loss": 0.5922021269798279, + "eval_mean_token_accuracy": 0.8346439617317777, + "eval_num_tokens": 281241.0, + "eval_runtime": 86.1551, + "eval_samples_per_second": 15.96, + "eval_steps_per_second": 1.996, + "step": 120 + }, + { + "entropy": 0.6663189359009266, + "epoch": 0.34869240348692404, + "grad_norm": 0.9528499841690063, + "learning_rate": 7.824090674661818e-05, + "loss": 0.5891091346740722, + "mean_token_accuracy": 0.832152470946312, + "num_tokens": 327393.0, + "step": 140 + }, + { + "epoch": 0.34869240348692404, + "eval_entropy": 0.6398407208711602, + "eval_loss": 0.5859636664390564, + "eval_mean_token_accuracy": 0.8372074996316156, + "eval_num_tokens": 327393.0, + "eval_runtime": 86.2706, + "eval_samples_per_second": 15.938, + "eval_steps_per_second": 1.994, + "step": 140 + }, + { + "entropy": 0.64859763905406, + "epoch": 0.398505603985056, + "grad_norm": 0.8468204140663147, + "learning_rate": 8.949859117059201e-05, + "loss": 0.569426441192627, + "mean_token_accuracy": 0.8401990942656994, + "num_tokens": 373834.0, + "step": 160 + }, + { + "epoch": 0.398505603985056, + "eval_entropy": 0.6381674285891444, + "eval_loss": 0.5744525790214539, + "eval_mean_token_accuracy": 0.838626817908398, + "eval_num_tokens": 373834.0, + "eval_runtime": 86.2848, + "eval_samples_per_second": 15.936, + "eval_steps_per_second": 1.993, + "step": 160 + }, + { + "entropy": 0.6432608783245086, + "epoch": 0.44831880448318806, + "grad_norm": 0.8765804767608643, + "learning_rate": 0.00010075627559456587, + "loss": 0.5687318801879883, + "mean_token_accuracy": 0.839249350130558, + "num_tokens": 422572.0, + "step": 180 + }, + { + "epoch": 0.44831880448318806, + "eval_entropy": 0.6047098288355872, + "eval_loss": 0.5679298043251038, + "eval_mean_token_accuracy": 0.8410577181466791, + "eval_num_tokens": 422572.0, + "eval_runtime": 86.5879, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.986, + "step": 180 + }, + { + "entropy": 0.6440276011824608, + "epoch": 0.49813200498132004, + "grad_norm": 0.9576020240783691, + "learning_rate": 0.00011201396001853971, + "loss": 0.5828506469726562, + "mean_token_accuracy": 0.837553184479475, + "num_tokens": 471879.0, + "step": 200 + }, + { + "epoch": 0.49813200498132004, + "eval_entropy": 0.6161119360909906, + "eval_loss": 0.5702911615371704, + "eval_mean_token_accuracy": 0.8407089398350827, + "eval_num_tokens": 471879.0, + "eval_runtime": 86.3341, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 200 + }, + { + "entropy": 0.6377195850014686, + "epoch": 0.547945205479452, + "grad_norm": 0.7212373614311218, + "learning_rate": 0.00012327164444251353, + "loss": 0.5702451229095459, + "mean_token_accuracy": 0.8397969007492065, + "num_tokens": 520984.0, + "step": 220 + }, + { + "epoch": 0.547945205479452, + "eval_entropy": 0.6080108886194784, + "eval_loss": 0.5633499622344971, + "eval_mean_token_accuracy": 0.8396634854549585, + "eval_num_tokens": 520984.0, + "eval_runtime": 86.4945, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 220 + }, + { + "entropy": 0.6287345830351114, + "epoch": 0.597758405977584, + "grad_norm": 0.848779022693634, + "learning_rate": 0.00013452932886648739, + "loss": 0.5506546020507812, + "mean_token_accuracy": 0.8438881888985634, + "num_tokens": 566596.0, + "step": 240 + }, + { + "epoch": 0.597758405977584, + "eval_entropy": 0.6307531505130058, + "eval_loss": 0.5573338270187378, + "eval_mean_token_accuracy": 0.8431362606758295, + "eval_num_tokens": 566596.0, + "eval_runtime": 86.3535, + "eval_samples_per_second": 15.923, + "eval_steps_per_second": 1.992, + "step": 240 + }, + { + "entropy": 0.6223786748945713, + "epoch": 0.6475716064757161, + "grad_norm": 0.7316951751708984, + "learning_rate": 0.0001457870132904612, + "loss": 0.5495625972747803, + "mean_token_accuracy": 0.8440376669168472, + "num_tokens": 613603.0, + "step": 260 + }, + { + "epoch": 0.6475716064757161, + "eval_entropy": 0.623454462476941, + "eval_loss": 0.5619264245033264, + "eval_mean_token_accuracy": 0.8431175777385401, + "eval_num_tokens": 613603.0, + "eval_runtime": 86.2008, + "eval_samples_per_second": 15.951, + "eval_steps_per_second": 1.995, + "step": 260 + }, + { + "entropy": 0.6281675305217505, + "epoch": 0.6973848069738481, + "grad_norm": 0.7639564871788025, + "learning_rate": 0.00015704469771443506, + "loss": 0.5604369163513183, + "mean_token_accuracy": 0.8401600055396556, + "num_tokens": 658565.0, + "step": 280 + }, + { + "epoch": 0.6973848069738481, + "eval_entropy": 0.63416675980701, + "eval_loss": 0.5612760782241821, + "eval_mean_token_accuracy": 0.842435666294985, + "eval_num_tokens": 658565.0, + "eval_runtime": 86.25, + "eval_samples_per_second": 15.942, + "eval_steps_per_second": 1.994, + "step": 280 + }, + { + "entropy": 0.6427909277379513, + "epoch": 0.7471980074719801, + "grad_norm": 0.6475813388824463, + "learning_rate": 0.0001683023821384089, + "loss": 0.573763370513916, + "mean_token_accuracy": 0.8370340794324875, + "num_tokens": 705680.0, + "step": 300 + }, + { + "epoch": 0.7471980074719801, + "eval_entropy": 0.6231539840268534, + "eval_loss": 0.5566866397857666, + "eval_mean_token_accuracy": 0.844177934319474, + "eval_num_tokens": 705680.0, + "eval_runtime": 86.4858, + "eval_samples_per_second": 15.899, + "eval_steps_per_second": 1.989, + "step": 300 + }, + { + "entropy": 0.6226776849478484, + "epoch": 0.797011207970112, + "grad_norm": 0.8886699676513672, + "learning_rate": 0.00017956006656238274, + "loss": 0.558210802078247, + "mean_token_accuracy": 0.84083157107234, + "num_tokens": 752616.0, + "step": 320 + }, + { + "epoch": 0.797011207970112, + "eval_entropy": 0.6066981683983359, + "eval_loss": 0.5585207939147949, + "eval_mean_token_accuracy": 0.8423153311014175, + "eval_num_tokens": 752616.0, + "eval_runtime": 86.3463, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 320 + }, + { + "entropy": 0.6249004438519478, + "epoch": 0.8468244084682441, + "grad_norm": 0.8791211843490601, + "learning_rate": 0.00019081775098635657, + "loss": 0.5603597164154053, + "mean_token_accuracy": 0.8420463085174561, + "num_tokens": 797151.0, + "step": 340 + }, + { + "epoch": 0.8468244084682441, + "eval_entropy": 0.6082247584018596, + "eval_loss": 0.5616299510002136, + "eval_mean_token_accuracy": 0.8431286801432454, + "eval_num_tokens": 797151.0, + "eval_runtime": 86.1253, + "eval_samples_per_second": 15.965, + "eval_steps_per_second": 1.997, + "step": 340 + }, + { + "entropy": 0.6362396612763405, + "epoch": 0.8966376089663761, + "grad_norm": 0.8606319427490234, + "learning_rate": 0.0002020754354103304, + "loss": 0.5735773563385009, + "mean_token_accuracy": 0.8371490836143494, + "num_tokens": 843585.0, + "step": 360 + }, + { + "epoch": 0.8966376089663761, + "eval_entropy": 0.6492362072648004, + "eval_loss": 0.5646467804908752, + "eval_mean_token_accuracy": 0.8415517574825953, + "eval_num_tokens": 843585.0, + "eval_runtime": 86.3351, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 360 + }, + { + "entropy": 0.638665035739541, + "epoch": 0.9464508094645081, + "grad_norm": 0.7773950099945068, + "learning_rate": 0.00021333311983430425, + "loss": 0.5820859909057617, + "mean_token_accuracy": 0.8372561208903789, + "num_tokens": 889842.0, + "step": 380 + }, + { + "epoch": 0.9464508094645081, + "eval_entropy": 0.6434498637221581, + "eval_loss": 0.5645168423652649, + "eval_mean_token_accuracy": 0.8420382481674815, + "eval_num_tokens": 889842.0, + "eval_runtime": 86.1216, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 380 + }, + { + "entropy": 0.6316851265728474, + "epoch": 0.9962640099626401, + "grad_norm": 1.6120579242706299, + "learning_rate": 0.00022459080425827807, + "loss": 0.5637502670288086, + "mean_token_accuracy": 0.8386227294802666, + "num_tokens": 935589.0, + "step": 400 + }, + { + "epoch": 0.9962640099626401, + "eval_entropy": 0.6469012776086497, + "eval_loss": 0.5758090615272522, + "eval_mean_token_accuracy": 0.8397158470957778, + "eval_num_tokens": 935589.0, + "eval_runtime": 86.6139, + "eval_samples_per_second": 15.875, + "eval_steps_per_second": 1.986, + "step": 400 + }, + { + "entropy": 0.5894816922835815, + "epoch": 1.0448318804483188, + "grad_norm": 1.1616325378417969, + "learning_rate": 0.00022626713048053178, + "loss": 0.5316025257110596, + "mean_token_accuracy": 0.8466163017810919, + "num_tokens": 980589.0, + "step": 420 + }, + { + "epoch": 1.0448318804483188, + "eval_entropy": 0.5860798164855602, + "eval_loss": 0.5777581930160522, + "eval_mean_token_accuracy": 0.8396938103576039, + "eval_num_tokens": 980589.0, + "eval_runtime": 86.1449, + "eval_samples_per_second": 15.961, + "eval_steps_per_second": 1.997, + "step": 420 + }, + { + "entropy": 0.5818420693278312, + "epoch": 1.0946450809464507, + "grad_norm": 0.7999453544616699, + "learning_rate": 0.00022622107023288778, + "loss": 0.5221010208129883, + "mean_token_accuracy": 0.8474301159381866, + "num_tokens": 1027852.0, + "step": 440 + }, + { + "epoch": 1.0946450809464507, + "eval_entropy": 0.5783926014636838, + "eval_loss": 0.5700300931930542, + "eval_mean_token_accuracy": 0.8430753537388735, + "eval_num_tokens": 1027852.0, + "eval_runtime": 86.5308, + "eval_samples_per_second": 15.89, + "eval_steps_per_second": 1.988, + "step": 440 + }, + { + "entropy": 0.5612493887543678, + "epoch": 1.1444582814445827, + "grad_norm": 1.015687346458435, + "learning_rate": 0.00022614090619491568, + "loss": 0.5084867000579834, + "mean_token_accuracy": 0.8495561093091964, + "num_tokens": 1077649.0, + "step": 460 + }, + { + "epoch": 1.1444582814445827, + "eval_entropy": 0.5841563874205877, + "eval_loss": 0.5693665742874146, + "eval_mean_token_accuracy": 0.8427817298229351, + "eval_num_tokens": 1077649.0, + "eval_runtime": 86.5256, + "eval_samples_per_second": 15.891, + "eval_steps_per_second": 1.988, + "step": 460 + }, + { + "entropy": 0.5828216474503278, + "epoch": 1.1942714819427147, + "grad_norm": 1.9750930070877075, + "learning_rate": 0.00022602666254299594, + "loss": 0.5180017948150635, + "mean_token_accuracy": 0.8515685826539994, + "num_tokens": 1124872.0, + "step": 480 + }, + { + "epoch": 1.1942714819427147, + "eval_entropy": 0.5806607044366903, + "eval_loss": 0.5804352760314941, + "eval_mean_token_accuracy": 0.8413014668364858, + "eval_num_tokens": 1124872.0, + "eval_runtime": 86.1199, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 480 + }, + { + "entropy": 0.5926914308220148, + "epoch": 1.244084682440847, + "grad_norm": 0.8917353749275208, + "learning_rate": 0.0002258783737314558, + "loss": 0.528910779953003, + "mean_token_accuracy": 0.8486074328422546, + "num_tokens": 1168698.0, + "step": 500 + }, + { + "epoch": 1.244084682440847, + "eval_entropy": 0.5593361884009006, + "eval_loss": 0.5675153732299805, + "eval_mean_token_accuracy": 0.8433507802181466, + "eval_num_tokens": 1168698.0, + "eval_runtime": 86.7289, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 500 + }, + { + "entropy": 0.5865630559623242, + "epoch": 1.293897882938979, + "grad_norm": 0.7482362985610962, + "learning_rate": 0.00022569608448217823, + "loss": 0.5250466823577881, + "mean_token_accuracy": 0.8477916084229946, + "num_tokens": 1216679.0, + "step": 520 + }, + { + "epoch": 1.293897882938979, + "eval_entropy": 0.543057840230853, + "eval_loss": 0.5671008229255676, + "eval_mean_token_accuracy": 0.8428726016088973, + "eval_num_tokens": 1216679.0, + "eval_runtime": 86.3403, + "eval_samples_per_second": 15.925, + "eval_steps_per_second": 1.992, + "step": 520 + }, + { + "entropy": 0.5870206747204065, + "epoch": 1.3437110834371109, + "grad_norm": 0.9473814964294434, + "learning_rate": 0.00022547984977111448, + "loss": 0.5252370834350586, + "mean_token_accuracy": 0.8468369916081429, + "num_tokens": 1261365.0, + "step": 540 + }, + { + "epoch": 1.3437110834371109, + "eval_entropy": 0.590982622878496, + "eval_loss": 0.5676343441009521, + "eval_mean_token_accuracy": 0.8429348746011424, + "eval_num_tokens": 1261365.0, + "eval_runtime": 86.5168, + "eval_samples_per_second": 15.893, + "eval_steps_per_second": 1.988, + "step": 540 + }, + { + "entropy": 0.5785854265093804, + "epoch": 1.3935242839352429, + "grad_norm": 0.9353351593017578, + "learning_rate": 0.0002252297348117042, + "loss": 0.5304938316345215, + "mean_token_accuracy": 0.8463383808732032, + "num_tokens": 1306879.0, + "step": 560 + }, + { + "epoch": 1.3935242839352429, + "eval_entropy": 0.6099918867612995, + "eval_loss": 0.5620437860488892, + "eval_mean_token_accuracy": 0.8430728347495545, + "eval_num_tokens": 1306879.0, + "eval_runtime": 86.7741, + "eval_samples_per_second": 15.846, + "eval_steps_per_second": 1.982, + "step": 560 + }, + { + "entropy": 0.5768801040947438, + "epoch": 1.4433374844333748, + "grad_norm": 0.9198738932609558, + "learning_rate": 0.0002249458150352077, + "loss": 0.520513391494751, + "mean_token_accuracy": 0.8487689301371575, + "num_tokens": 1353534.0, + "step": 580 + }, + { + "epoch": 1.4433374844333748, + "eval_entropy": 0.6349420670506566, + "eval_loss": 0.5645340085029602, + "eval_mean_token_accuracy": 0.8447844597489335, + "eval_num_tokens": 1353534.0, + "eval_runtime": 86.3257, + "eval_samples_per_second": 15.928, + "eval_steps_per_second": 1.992, + "step": 580 + }, + { + "entropy": 0.5822233572602272, + "epoch": 1.4931506849315068, + "grad_norm": 0.832811176776886, + "learning_rate": 0.0002246281760679571, + "loss": 0.5295282363891601, + "mean_token_accuracy": 0.8504064798355102, + "num_tokens": 1399537.0, + "step": 600 + }, + { + "epoch": 1.4931506849315068, + "eval_entropy": 0.5829724387027496, + "eval_loss": 0.5612193942070007, + "eval_mean_token_accuracy": 0.8449643853791925, + "eval_num_tokens": 1399537.0, + "eval_runtime": 86.6617, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 600 + }, + { + "entropy": 0.571855777129531, + "epoch": 1.5429638854296388, + "grad_norm": 0.7665547728538513, + "learning_rate": 0.00022427691370553263, + "loss": 0.5187931060791016, + "mean_token_accuracy": 0.8534420043230057, + "num_tokens": 1448422.0, + "step": 620 + }, + { + "epoch": 1.5429638854296388, + "eval_entropy": 0.5623592240519302, + "eval_loss": 0.5575760006904602, + "eval_mean_token_accuracy": 0.8468210229346919, + "eval_num_tokens": 1448422.0, + "eval_runtime": 86.6324, + "eval_samples_per_second": 15.872, + "eval_steps_per_second": 1.985, + "step": 620 + }, + { + "entropy": 0.5740394659340382, + "epoch": 1.592777085927771, + "grad_norm": 0.6500429511070251, + "learning_rate": 0.00022389213388387174, + "loss": 0.5283198833465577, + "mean_token_accuracy": 0.8502798482775689, + "num_tokens": 1495009.0, + "step": 640 + }, + { + "epoch": 1.592777085927771, + "eval_entropy": 0.5548852207355721, + "eval_loss": 0.5561797022819519, + "eval_mean_token_accuracy": 0.8452786498291548, + "eval_num_tokens": 1495009.0, + "eval_runtime": 86.5205, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 640 + }, + { + "entropy": 0.6020145989954472, + "epoch": 1.6425902864259028, + "grad_norm": 0.7056867480278015, + "learning_rate": 0.00022347395264732053, + "loss": 0.5400049209594726, + "mean_token_accuracy": 0.8447613954544068, + "num_tokens": 1536932.0, + "step": 660 + }, + { + "epoch": 1.6425902864259028, + "eval_entropy": 0.5618055154417836, + "eval_loss": 0.556106686592102, + "eval_mean_token_accuracy": 0.8465680112672407, + "eval_num_tokens": 1536932.0, + "eval_runtime": 86.2971, + "eval_samples_per_second": 15.933, + "eval_steps_per_second": 1.993, + "step": 660 + }, + { + "entropy": 0.5665927153080702, + "epoch": 1.692403486924035, + "grad_norm": 0.5987663865089417, + "learning_rate": 0.00022302249611363625, + "loss": 0.5143643856048584, + "mean_token_accuracy": 0.8529589556157589, + "num_tokens": 1585718.0, + "step": 680 + }, + { + "epoch": 1.692403486924035, + "eval_entropy": 0.568248552118623, + "eval_loss": 0.5476346015930176, + "eval_mean_token_accuracy": 0.8476775434128073, + "eval_num_tokens": 1585718.0, + "eval_runtime": 86.9583, + "eval_samples_per_second": 15.812, + "eval_steps_per_second": 1.978, + "step": 680 + }, + { + "entropy": 0.5673687808215618, + "epoch": 1.7422166874221667, + "grad_norm": 0.735261857509613, + "learning_rate": 0.00022253790043595193, + "loss": 0.509885597229004, + "mean_token_accuracy": 0.8537046857178211, + "num_tokens": 1635718.0, + "step": 700 + }, + { + "epoch": 1.7422166874221667, + "eval_entropy": 0.5616967284748721, + "eval_loss": 0.5439274311065674, + "eval_mean_token_accuracy": 0.8488946217437123, + "eval_num_tokens": 1635718.0, + "eval_runtime": 86.0604, + "eval_samples_per_second": 15.977, + "eval_steps_per_second": 1.999, + "step": 700 + }, + { + "entropy": 0.5529541682451964, + "epoch": 1.792029887920299, + "grad_norm": 0.7014835476875305, + "learning_rate": 0.00022202031176171442, + "loss": 0.5078992366790771, + "mean_token_accuracy": 0.8525233261287213, + "num_tokens": 1681291.0, + "step": 720 + }, + { + "epoch": 1.792029887920299, + "eval_entropy": 0.5827173320359962, + "eval_loss": 0.5419450402259827, + "eval_mean_token_accuracy": 0.8477318609176681, + "eval_num_tokens": 1681291.0, + "eval_runtime": 85.2984, + "eval_samples_per_second": 16.12, + "eval_steps_per_second": 2.016, + "step": 720 + }, + { + "entropy": 0.5755720350891351, + "epoch": 1.841843088418431, + "grad_norm": 0.705613911151886, + "learning_rate": 0.00022146988618860824, + "loss": 0.5181350708007812, + "mean_token_accuracy": 0.8467609457671642, + "num_tokens": 1729102.0, + "step": 740 + }, + { + "epoch": 1.841843088418431, + "eval_entropy": 0.5743971356125765, + "eval_loss": 0.5415896773338318, + "eval_mean_token_accuracy": 0.847328585940738, + "eval_num_tokens": 1729102.0, + "eval_runtime": 85.5602, + "eval_samples_per_second": 16.071, + "eval_steps_per_second": 2.01, + "step": 740 + }, + { + "entropy": 0.561330484598875, + "epoch": 1.891656288916563, + "grad_norm": 0.6722865700721741, + "learning_rate": 0.0002208867897174789, + "loss": 0.499837589263916, + "mean_token_accuracy": 0.8518734864890576, + "num_tokens": 1773578.0, + "step": 760 + }, + { + "epoch": 1.891656288916563, + "eval_entropy": 0.5865232653396074, + "eval_loss": 0.5437926650047302, + "eval_mean_token_accuracy": 0.8450997017843779, + "eval_num_tokens": 1773578.0, + "eval_runtime": 86.4116, + "eval_samples_per_second": 15.912, + "eval_steps_per_second": 1.99, + "step": 760 + }, + { + "entropy": 0.547389242425561, + "epoch": 1.9414694894146949, + "grad_norm": 0.7935577034950256, + "learning_rate": 0.00022027119820226907, + "loss": 0.4977591514587402, + "mean_token_accuracy": 0.8539491161704064, + "num_tokens": 1821725.0, + "step": 780 + }, + { + "epoch": 1.9414694894146949, + "eval_entropy": 0.5290903090391048, + "eval_loss": 0.5409526824951172, + "eval_mean_token_accuracy": 0.8497545698354411, + "eval_num_tokens": 1821725.0, + "eval_runtime": 86.7262, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 780 + }, + { + "entropy": 0.5687909748405218, + "epoch": 1.9912826899128269, + "grad_norm": 0.6180546283721924, + "learning_rate": 0.00021962329729698345, + "loss": 0.5109643459320068, + "mean_token_accuracy": 0.8521598495543004, + "num_tokens": 1868431.0, + "step": 800 + }, + { + "epoch": 1.9912826899128269, + "eval_entropy": 0.5503541858390321, + "eval_loss": 0.5361555218696594, + "eval_mean_token_accuracy": 0.8510884285666221, + "eval_num_tokens": 1868431.0, + "eval_runtime": 86.3339, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 800 + }, + { + "entropy": 0.4739728841261986, + "epoch": 2.0398505603985058, + "grad_norm": 0.8058829307556152, + "learning_rate": 0.0002189432823996982, + "loss": 0.4204097747802734, + "mean_token_accuracy": 0.8728981889211215, + "num_tokens": 1915280.0, + "step": 820 + }, + { + "epoch": 2.0398505603985058, + "eval_entropy": 0.5077334992414297, + "eval_loss": 0.5531114339828491, + "eval_mean_token_accuracy": 0.8489257208136625, + "eval_num_tokens": 1915280.0, + "eval_runtime": 86.4801, + "eval_samples_per_second": 15.9, + "eval_steps_per_second": 1.989, + "step": 820 + }, + { + "entropy": 0.4594309840351343, + "epoch": 2.0896637608966375, + "grad_norm": 0.6906896829605103, + "learning_rate": 0.0002182313585936314, + "loss": 0.4071959495544434, + "mean_token_accuracy": 0.8732857562601566, + "num_tokens": 1965306.0, + "step": 840 + }, + { + "epoch": 2.0896637608966375, + "eval_entropy": 0.49850136994622474, + "eval_loss": 0.5486204624176025, + "eval_mean_token_accuracy": 0.8507991450470548, + "eval_num_tokens": 1965306.0, + "eval_runtime": 86.3364, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 840 + }, + { + "entropy": 0.4881629109382629, + "epoch": 2.1394769613947697, + "grad_norm": 0.6343470215797424, + "learning_rate": 0.0002174877405852928, + "loss": 0.41669540405273436, + "mean_token_accuracy": 0.8711295068264008, + "num_tokens": 2008562.0, + "step": 860 + }, + { + "epoch": 2.1394769613947697, + "eval_entropy": 0.49155513924914734, + "eval_loss": 0.555109441280365, + "eval_mean_token_accuracy": 0.8496399400539176, + "eval_num_tokens": 2008562.0, + "eval_runtime": 86.3295, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 860 + }, + { + "entropy": 0.4648668970912695, + "epoch": 2.1892901618929015, + "grad_norm": 0.8014165163040161, + "learning_rate": 0.00021671265263973133, + "loss": 0.4110250473022461, + "mean_token_accuracy": 0.8754166305065155, + "num_tokens": 2056474.0, + "step": 880 + }, + { + "epoch": 2.1892901618929015, + "eval_entropy": 0.4909258722219356, + "eval_loss": 0.5539511442184448, + "eval_mean_token_accuracy": 0.8492401502160138, + "eval_num_tokens": 2056474.0, + "eval_runtime": 86.3468, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 880 + }, + { + "entropy": 0.4824485514312983, + "epoch": 2.2391033623910337, + "grad_norm": 0.6665191054344177, + "learning_rate": 0.00021590632851289967, + "loss": 0.4181404113769531, + "mean_token_accuracy": 0.8726993151009083, + "num_tokens": 2103543.0, + "step": 900 + }, + { + "epoch": 2.2391033623910337, + "eval_entropy": 0.4986876940657926, + "eval_loss": 0.547695517539978, + "eval_mean_token_accuracy": 0.8501384708770486, + "eval_num_tokens": 2103543.0, + "eval_runtime": 86.3838, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 900 + }, + { + "entropy": 0.4751896943897009, + "epoch": 2.2889165628891655, + "grad_norm": 0.81158047914505, + "learning_rate": 0.00021506901138115678, + "loss": 0.40689678192138673, + "mean_token_accuracy": 0.8745221219956875, + "num_tokens": 2147861.0, + "step": 920 + }, + { + "epoch": 2.2889165628891655, + "eval_entropy": 0.507153491121392, + "eval_loss": 0.5501641631126404, + "eval_mean_token_accuracy": 0.8495670116918032, + "eval_num_tokens": 2147861.0, + "eval_runtime": 86.0912, + "eval_samples_per_second": 15.971, + "eval_steps_per_second": 1.998, + "step": 920 + }, + { + "entropy": 0.4873133715242147, + "epoch": 2.3387297633872977, + "grad_norm": 0.7218056321144104, + "learning_rate": 0.0002142009537679292, + "loss": 0.42701358795166017, + "mean_token_accuracy": 0.8695114746689796, + "num_tokens": 2190561.0, + "step": 940 + }, + { + "epoch": 2.3387297633872977, + "eval_entropy": 0.5202612736543943, + "eval_loss": 0.5491839051246643, + "eval_mean_token_accuracy": 0.8494071208460386, + "eval_num_tokens": 2190561.0, + "eval_runtime": 86.1142, + "eval_samples_per_second": 15.967, + "eval_steps_per_second": 1.997, + "step": 940 + }, + { + "entropy": 0.4762951169162989, + "epoch": 2.3885429638854294, + "grad_norm": 0.7194424867630005, + "learning_rate": 0.0002133024174675534, + "loss": 0.42299847602844237, + "mean_token_accuracy": 0.8709790132939815, + "num_tokens": 2239412.0, + "step": 960 + }, + { + "epoch": 2.3885429638854294, + "eval_entropy": 0.4899340462546016, + "eval_loss": 0.5522511601448059, + "eval_mean_token_accuracy": 0.8492208258357159, + "eval_num_tokens": 2239412.0, + "eval_runtime": 86.463, + "eval_samples_per_second": 15.903, + "eval_steps_per_second": 1.989, + "step": 960 + }, + { + "entropy": 0.49650347977876663, + "epoch": 2.4383561643835616, + "grad_norm": 0.8406022787094116, + "learning_rate": 0.0002123736734663221, + "loss": 0.4275330066680908, + "mean_token_accuracy": 0.8670595556497573, + "num_tokens": 2286283.0, + "step": 980 + }, + { + "epoch": 2.4383561643835616, + "eval_entropy": 0.49691385654515996, + "eval_loss": 0.5491269826889038, + "eval_mean_token_accuracy": 0.850309816210769, + "eval_num_tokens": 2286283.0, + "eval_runtime": 86.17, + "eval_samples_per_second": 15.957, + "eval_steps_per_second": 1.996, + "step": 980 + }, + { + "entropy": 0.48843890577554705, + "epoch": 2.488169364881694, + "grad_norm": 0.9082473516464233, + "learning_rate": 0.00021141500186075868, + "loss": 0.4309722423553467, + "mean_token_accuracy": 0.8686766296625137, + "num_tokens": 2333733.0, + "step": 1000 + }, + { + "epoch": 2.488169364881694, + "eval_entropy": 0.5543508351195691, + "eval_loss": 0.5478800535202026, + "eval_mean_token_accuracy": 0.8478029522784921, + "eval_num_tokens": 2333733.0, + "eval_runtime": 86.3835, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 1000 + }, + { + "entropy": 0.4777219031006098, + "epoch": 2.5379825653798256, + "grad_norm": 0.7448089122772217, + "learning_rate": 0.0002104266917731438, + "loss": 0.423325252532959, + "mean_token_accuracy": 0.8706337086856365, + "num_tokens": 2384270.0, + "step": 1020 + }, + { + "epoch": 2.5379825653798256, + "eval_entropy": 0.49857561550168106, + "eval_loss": 0.5511948466300964, + "eval_mean_token_accuracy": 0.8502220289651737, + "eval_num_tokens": 2384270.0, + "eval_runtime": 86.5399, + "eval_samples_per_second": 15.889, + "eval_steps_per_second": 1.988, + "step": 1020 + }, + { + "entropy": 0.4844174191355705, + "epoch": 2.587795765877958, + "grad_norm": 0.794029176235199, + "learning_rate": 0.00020940904126432, + "loss": 0.4176753044128418, + "mean_token_accuracy": 0.873535567522049, + "num_tokens": 2428036.0, + "step": 1040 + }, + { + "epoch": 2.587795765877958, + "eval_entropy": 0.485467542222766, + "eval_loss": 0.5539286732673645, + "eval_mean_token_accuracy": 0.8495475081510322, + "eval_num_tokens": 2428036.0, + "eval_runtime": 86.135, + "eval_samples_per_second": 15.963, + "eval_steps_per_second": 1.997, + "step": 1040 + }, + { + "entropy": 0.49070929251611234, + "epoch": 2.6376089663760895, + "grad_norm": 0.7558256983757019, + "learning_rate": 0.0002083623572438007, + "loss": 0.42867293357849123, + "mean_token_accuracy": 0.8696666076779366, + "num_tokens": 2476815.0, + "step": 1060 + }, + { + "epoch": 2.6376089663760895, + "eval_entropy": 0.490822730889154, + "eval_loss": 0.5434785485267639, + "eval_mean_token_accuracy": 0.850568296950917, + "eval_num_tokens": 2476815.0, + "eval_runtime": 86.4933, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 1060 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.045743734380032e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/chat_template.jinja b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/tokenizer_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/trainer_state.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c1462c7cf2ca30d2ffd6b28c99807bb03c778216 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1080/trainer_state.json @@ -0,0 +1,1168 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.6874221668742218, + "eval_steps": 20, + "global_step": 1080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.955029806494713, + "epoch": 0.049813200498132, + "grad_norm": 3.020533561706543, + "learning_rate": 1.0694800202775147e-05, + "loss": 1.7107986450195312, + "mean_token_accuracy": 0.6487608112394809, + "num_tokens": 46794.0, + "step": 20 + }, + { + "epoch": 0.049813200498132, + "eval_entropy": 1.3144892034835594, + "eval_loss": 1.1198534965515137, + "eval_mean_token_accuracy": 0.7460246955932572, + "eval_num_tokens": 46794.0, + "eval_runtime": 87.0565, + "eval_samples_per_second": 15.794, + "eval_steps_per_second": 1.976, + "step": 20 + }, + { + "entropy": 1.0063214391469955, + "epoch": 0.099626400996264, + "grad_norm": 1.572906494140625, + "learning_rate": 2.1952484626748985e-05, + "loss": 0.8663722991943359, + "mean_token_accuracy": 0.7779282338917255, + "num_tokens": 90754.0, + "step": 40 + }, + { + "epoch": 0.099626400996264, + "eval_entropy": 0.7921617945959402, + "eval_loss": 0.7062025666236877, + "eval_mean_token_accuracy": 0.8100443180910376, + "eval_num_tokens": 90754.0, + "eval_runtime": 86.5189, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 40 + }, + { + "entropy": 0.7682028576731682, + "epoch": 0.149439601494396, + "grad_norm": 1.3003711700439453, + "learning_rate": 3.3210169050722824e-05, + "loss": 0.673183822631836, + "mean_token_accuracy": 0.8182129614055157, + "num_tokens": 137472.0, + "step": 60 + }, + { + "epoch": 0.149439601494396, + "eval_entropy": 0.7059133584762729, + "eval_loss": 0.6481946706771851, + "eval_mean_token_accuracy": 0.8227418761613757, + "eval_num_tokens": 137472.0, + "eval_runtime": 86.5098, + "eval_samples_per_second": 15.894, + "eval_steps_per_second": 1.988, + "step": 60 + }, + { + "entropy": 0.7029960259795189, + "epoch": 0.199252801992528, + "grad_norm": 1.3664201498031616, + "learning_rate": 4.4467853474696664e-05, + "loss": 0.6354611873626709, + "mean_token_accuracy": 0.8243416830897331, + "num_tokens": 187408.0, + "step": 80 + }, + { + "epoch": 0.199252801992528, + "eval_entropy": 0.6867308004997498, + "eval_loss": 0.6179807186126709, + "eval_mean_token_accuracy": 0.8302594811417335, + "eval_num_tokens": 187408.0, + "eval_runtime": 86.3969, + "eval_samples_per_second": 15.915, + "eval_steps_per_second": 1.991, + "step": 80 + }, + { + "entropy": 0.6764581337571144, + "epoch": 0.24906600249066002, + "grad_norm": 0.9815880656242371, + "learning_rate": 5.57255378986705e-05, + "loss": 0.5988658905029297, + "mean_token_accuracy": 0.8329168625175953, + "num_tokens": 234197.0, + "step": 100 + }, + { + "epoch": 0.24906600249066002, + "eval_entropy": 0.6790881479202315, + "eval_loss": 0.5998476147651672, + "eval_mean_token_accuracy": 0.8318756420251935, + "eval_num_tokens": 234197.0, + "eval_runtime": 86.6653, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 100 + }, + { + "entropy": 0.6744543805718421, + "epoch": 0.298879202988792, + "grad_norm": 0.932099461555481, + "learning_rate": 6.698322232264434e-05, + "loss": 0.5991750717163086, + "mean_token_accuracy": 0.8304223112761975, + "num_tokens": 281241.0, + "step": 120 + }, + { + "epoch": 0.298879202988792, + "eval_entropy": 0.6813044282932614, + "eval_loss": 0.5922021269798279, + "eval_mean_token_accuracy": 0.8346439617317777, + "eval_num_tokens": 281241.0, + "eval_runtime": 86.1551, + "eval_samples_per_second": 15.96, + "eval_steps_per_second": 1.996, + "step": 120 + }, + { + "entropy": 0.6663189359009266, + "epoch": 0.34869240348692404, + "grad_norm": 0.9528499841690063, + "learning_rate": 7.824090674661818e-05, + "loss": 0.5891091346740722, + "mean_token_accuracy": 0.832152470946312, + "num_tokens": 327393.0, + "step": 140 + }, + { + "epoch": 0.34869240348692404, + "eval_entropy": 0.6398407208711602, + "eval_loss": 0.5859636664390564, + "eval_mean_token_accuracy": 0.8372074996316156, + "eval_num_tokens": 327393.0, + "eval_runtime": 86.2706, + "eval_samples_per_second": 15.938, + "eval_steps_per_second": 1.994, + "step": 140 + }, + { + "entropy": 0.64859763905406, + "epoch": 0.398505603985056, + "grad_norm": 0.8468204140663147, + "learning_rate": 8.949859117059201e-05, + "loss": 0.569426441192627, + "mean_token_accuracy": 0.8401990942656994, + "num_tokens": 373834.0, + "step": 160 + }, + { + "epoch": 0.398505603985056, + "eval_entropy": 0.6381674285891444, + "eval_loss": 0.5744525790214539, + "eval_mean_token_accuracy": 0.838626817908398, + "eval_num_tokens": 373834.0, + "eval_runtime": 86.2848, + "eval_samples_per_second": 15.936, + "eval_steps_per_second": 1.993, + "step": 160 + }, + { + "entropy": 0.6432608783245086, + "epoch": 0.44831880448318806, + "grad_norm": 0.8765804767608643, + "learning_rate": 0.00010075627559456587, + "loss": 0.5687318801879883, + "mean_token_accuracy": 0.839249350130558, + "num_tokens": 422572.0, + "step": 180 + }, + { + "epoch": 0.44831880448318806, + "eval_entropy": 0.6047098288355872, + "eval_loss": 0.5679298043251038, + "eval_mean_token_accuracy": 0.8410577181466791, + "eval_num_tokens": 422572.0, + "eval_runtime": 86.5879, + "eval_samples_per_second": 15.88, + "eval_steps_per_second": 1.986, + "step": 180 + }, + { + "entropy": 0.6440276011824608, + "epoch": 0.49813200498132004, + "grad_norm": 0.9576020240783691, + "learning_rate": 0.00011201396001853971, + "loss": 0.5828506469726562, + "mean_token_accuracy": 0.837553184479475, + "num_tokens": 471879.0, + "step": 200 + }, + { + "epoch": 0.49813200498132004, + "eval_entropy": 0.6161119360909906, + "eval_loss": 0.5702911615371704, + "eval_mean_token_accuracy": 0.8407089398350827, + "eval_num_tokens": 471879.0, + "eval_runtime": 86.3341, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 200 + }, + { + "entropy": 0.6377195850014686, + "epoch": 0.547945205479452, + "grad_norm": 0.7212373614311218, + "learning_rate": 0.00012327164444251353, + "loss": 0.5702451229095459, + "mean_token_accuracy": 0.8397969007492065, + "num_tokens": 520984.0, + "step": 220 + }, + { + "epoch": 0.547945205479452, + "eval_entropy": 0.6080108886194784, + "eval_loss": 0.5633499622344971, + "eval_mean_token_accuracy": 0.8396634854549585, + "eval_num_tokens": 520984.0, + "eval_runtime": 86.4945, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 220 + }, + { + "entropy": 0.6287345830351114, + "epoch": 0.597758405977584, + "grad_norm": 0.848779022693634, + "learning_rate": 0.00013452932886648739, + "loss": 0.5506546020507812, + "mean_token_accuracy": 0.8438881888985634, + "num_tokens": 566596.0, + "step": 240 + }, + { + "epoch": 0.597758405977584, + "eval_entropy": 0.6307531505130058, + "eval_loss": 0.5573338270187378, + "eval_mean_token_accuracy": 0.8431362606758295, + "eval_num_tokens": 566596.0, + "eval_runtime": 86.3535, + "eval_samples_per_second": 15.923, + "eval_steps_per_second": 1.992, + "step": 240 + }, + { + "entropy": 0.6223786748945713, + "epoch": 0.6475716064757161, + "grad_norm": 0.7316951751708984, + "learning_rate": 0.0001457870132904612, + "loss": 0.5495625972747803, + "mean_token_accuracy": 0.8440376669168472, + "num_tokens": 613603.0, + "step": 260 + }, + { + "epoch": 0.6475716064757161, + "eval_entropy": 0.623454462476941, + "eval_loss": 0.5619264245033264, + "eval_mean_token_accuracy": 0.8431175777385401, + "eval_num_tokens": 613603.0, + "eval_runtime": 86.2008, + "eval_samples_per_second": 15.951, + "eval_steps_per_second": 1.995, + "step": 260 + }, + { + "entropy": 0.6281675305217505, + "epoch": 0.6973848069738481, + "grad_norm": 0.7639564871788025, + "learning_rate": 0.00015704469771443506, + "loss": 0.5604369163513183, + "mean_token_accuracy": 0.8401600055396556, + "num_tokens": 658565.0, + "step": 280 + }, + { + "epoch": 0.6973848069738481, + "eval_entropy": 0.63416675980701, + "eval_loss": 0.5612760782241821, + "eval_mean_token_accuracy": 0.842435666294985, + "eval_num_tokens": 658565.0, + "eval_runtime": 86.25, + "eval_samples_per_second": 15.942, + "eval_steps_per_second": 1.994, + "step": 280 + }, + { + "entropy": 0.6427909277379513, + "epoch": 0.7471980074719801, + "grad_norm": 0.6475813388824463, + "learning_rate": 0.0001683023821384089, + "loss": 0.573763370513916, + "mean_token_accuracy": 0.8370340794324875, + "num_tokens": 705680.0, + "step": 300 + }, + { + "epoch": 0.7471980074719801, + "eval_entropy": 0.6231539840268534, + "eval_loss": 0.5566866397857666, + "eval_mean_token_accuracy": 0.844177934319474, + "eval_num_tokens": 705680.0, + "eval_runtime": 86.4858, + "eval_samples_per_second": 15.899, + "eval_steps_per_second": 1.989, + "step": 300 + }, + { + "entropy": 0.6226776849478484, + "epoch": 0.797011207970112, + "grad_norm": 0.8886699676513672, + "learning_rate": 0.00017956006656238274, + "loss": 0.558210802078247, + "mean_token_accuracy": 0.84083157107234, + "num_tokens": 752616.0, + "step": 320 + }, + { + "epoch": 0.797011207970112, + "eval_entropy": 0.6066981683983359, + "eval_loss": 0.5585207939147949, + "eval_mean_token_accuracy": 0.8423153311014175, + "eval_num_tokens": 752616.0, + "eval_runtime": 86.3463, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 320 + }, + { + "entropy": 0.6249004438519478, + "epoch": 0.8468244084682441, + "grad_norm": 0.8791211843490601, + "learning_rate": 0.00019081775098635657, + "loss": 0.5603597164154053, + "mean_token_accuracy": 0.8420463085174561, + "num_tokens": 797151.0, + "step": 340 + }, + { + "epoch": 0.8468244084682441, + "eval_entropy": 0.6082247584018596, + "eval_loss": 0.5616299510002136, + "eval_mean_token_accuracy": 0.8431286801432454, + "eval_num_tokens": 797151.0, + "eval_runtime": 86.1253, + "eval_samples_per_second": 15.965, + "eval_steps_per_second": 1.997, + "step": 340 + }, + { + "entropy": 0.6362396612763405, + "epoch": 0.8966376089663761, + "grad_norm": 0.8606319427490234, + "learning_rate": 0.0002020754354103304, + "loss": 0.5735773563385009, + "mean_token_accuracy": 0.8371490836143494, + "num_tokens": 843585.0, + "step": 360 + }, + { + "epoch": 0.8966376089663761, + "eval_entropy": 0.6492362072648004, + "eval_loss": 0.5646467804908752, + "eval_mean_token_accuracy": 0.8415517574825953, + "eval_num_tokens": 843585.0, + "eval_runtime": 86.3351, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 360 + }, + { + "entropy": 0.638665035739541, + "epoch": 0.9464508094645081, + "grad_norm": 0.7773950099945068, + "learning_rate": 0.00021333311983430425, + "loss": 0.5820859909057617, + "mean_token_accuracy": 0.8372561208903789, + "num_tokens": 889842.0, + "step": 380 + }, + { + "epoch": 0.9464508094645081, + "eval_entropy": 0.6434498637221581, + "eval_loss": 0.5645168423652649, + "eval_mean_token_accuracy": 0.8420382481674815, + "eval_num_tokens": 889842.0, + "eval_runtime": 86.1216, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 380 + }, + { + "entropy": 0.6316851265728474, + "epoch": 0.9962640099626401, + "grad_norm": 1.6120579242706299, + "learning_rate": 0.00022459080425827807, + "loss": 0.5637502670288086, + "mean_token_accuracy": 0.8386227294802666, + "num_tokens": 935589.0, + "step": 400 + }, + { + "epoch": 0.9962640099626401, + "eval_entropy": 0.6469012776086497, + "eval_loss": 0.5758090615272522, + "eval_mean_token_accuracy": 0.8397158470957778, + "eval_num_tokens": 935589.0, + "eval_runtime": 86.6139, + "eval_samples_per_second": 15.875, + "eval_steps_per_second": 1.986, + "step": 400 + }, + { + "entropy": 0.5894816922835815, + "epoch": 1.0448318804483188, + "grad_norm": 1.1616325378417969, + "learning_rate": 0.00022626713048053178, + "loss": 0.5316025257110596, + "mean_token_accuracy": 0.8466163017810919, + "num_tokens": 980589.0, + "step": 420 + }, + { + "epoch": 1.0448318804483188, + "eval_entropy": 0.5860798164855602, + "eval_loss": 0.5777581930160522, + "eval_mean_token_accuracy": 0.8396938103576039, + "eval_num_tokens": 980589.0, + "eval_runtime": 86.1449, + "eval_samples_per_second": 15.961, + "eval_steps_per_second": 1.997, + "step": 420 + }, + { + "entropy": 0.5818420693278312, + "epoch": 1.0946450809464507, + "grad_norm": 0.7999453544616699, + "learning_rate": 0.00022622107023288778, + "loss": 0.5221010208129883, + "mean_token_accuracy": 0.8474301159381866, + "num_tokens": 1027852.0, + "step": 440 + }, + { + "epoch": 1.0946450809464507, + "eval_entropy": 0.5783926014636838, + "eval_loss": 0.5700300931930542, + "eval_mean_token_accuracy": 0.8430753537388735, + "eval_num_tokens": 1027852.0, + "eval_runtime": 86.5308, + "eval_samples_per_second": 15.89, + "eval_steps_per_second": 1.988, + "step": 440 + }, + { + "entropy": 0.5612493887543678, + "epoch": 1.1444582814445827, + "grad_norm": 1.015687346458435, + "learning_rate": 0.00022614090619491568, + "loss": 0.5084867000579834, + "mean_token_accuracy": 0.8495561093091964, + "num_tokens": 1077649.0, + "step": 460 + }, + { + "epoch": 1.1444582814445827, + "eval_entropy": 0.5841563874205877, + "eval_loss": 0.5693665742874146, + "eval_mean_token_accuracy": 0.8427817298229351, + "eval_num_tokens": 1077649.0, + "eval_runtime": 86.5256, + "eval_samples_per_second": 15.891, + "eval_steps_per_second": 1.988, + "step": 460 + }, + { + "entropy": 0.5828216474503278, + "epoch": 1.1942714819427147, + "grad_norm": 1.9750930070877075, + "learning_rate": 0.00022602666254299594, + "loss": 0.5180017948150635, + "mean_token_accuracy": 0.8515685826539994, + "num_tokens": 1124872.0, + "step": 480 + }, + { + "epoch": 1.1942714819427147, + "eval_entropy": 0.5806607044366903, + "eval_loss": 0.5804352760314941, + "eval_mean_token_accuracy": 0.8413014668364858, + "eval_num_tokens": 1124872.0, + "eval_runtime": 86.1199, + "eval_samples_per_second": 15.966, + "eval_steps_per_second": 1.997, + "step": 480 + }, + { + "entropy": 0.5926914308220148, + "epoch": 1.244084682440847, + "grad_norm": 0.8917353749275208, + "learning_rate": 0.0002258783737314558, + "loss": 0.528910779953003, + "mean_token_accuracy": 0.8486074328422546, + "num_tokens": 1168698.0, + "step": 500 + }, + { + "epoch": 1.244084682440847, + "eval_entropy": 0.5593361884009006, + "eval_loss": 0.5675153732299805, + "eval_mean_token_accuracy": 0.8433507802181466, + "eval_num_tokens": 1168698.0, + "eval_runtime": 86.7289, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 500 + }, + { + "entropy": 0.5865630559623242, + "epoch": 1.293897882938979, + "grad_norm": 0.7482362985610962, + "learning_rate": 0.00022569608448217823, + "loss": 0.5250466823577881, + "mean_token_accuracy": 0.8477916084229946, + "num_tokens": 1216679.0, + "step": 520 + }, + { + "epoch": 1.293897882938979, + "eval_entropy": 0.543057840230853, + "eval_loss": 0.5671008229255676, + "eval_mean_token_accuracy": 0.8428726016088973, + "eval_num_tokens": 1216679.0, + "eval_runtime": 86.3403, + "eval_samples_per_second": 15.925, + "eval_steps_per_second": 1.992, + "step": 520 + }, + { + "entropy": 0.5870206747204065, + "epoch": 1.3437110834371109, + "grad_norm": 0.9473814964294434, + "learning_rate": 0.00022547984977111448, + "loss": 0.5252370834350586, + "mean_token_accuracy": 0.8468369916081429, + "num_tokens": 1261365.0, + "step": 540 + }, + { + "epoch": 1.3437110834371109, + "eval_entropy": 0.590982622878496, + "eval_loss": 0.5676343441009521, + "eval_mean_token_accuracy": 0.8429348746011424, + "eval_num_tokens": 1261365.0, + "eval_runtime": 86.5168, + "eval_samples_per_second": 15.893, + "eval_steps_per_second": 1.988, + "step": 540 + }, + { + "entropy": 0.5785854265093804, + "epoch": 1.3935242839352429, + "grad_norm": 0.9353351593017578, + "learning_rate": 0.0002252297348117042, + "loss": 0.5304938316345215, + "mean_token_accuracy": 0.8463383808732032, + "num_tokens": 1306879.0, + "step": 560 + }, + { + "epoch": 1.3935242839352429, + "eval_entropy": 0.6099918867612995, + "eval_loss": 0.5620437860488892, + "eval_mean_token_accuracy": 0.8430728347495545, + "eval_num_tokens": 1306879.0, + "eval_runtime": 86.7741, + "eval_samples_per_second": 15.846, + "eval_steps_per_second": 1.982, + "step": 560 + }, + { + "entropy": 0.5768801040947438, + "epoch": 1.4433374844333748, + "grad_norm": 0.9198738932609558, + "learning_rate": 0.0002249458150352077, + "loss": 0.520513391494751, + "mean_token_accuracy": 0.8487689301371575, + "num_tokens": 1353534.0, + "step": 580 + }, + { + "epoch": 1.4433374844333748, + "eval_entropy": 0.6349420670506566, + "eval_loss": 0.5645340085029602, + "eval_mean_token_accuracy": 0.8447844597489335, + "eval_num_tokens": 1353534.0, + "eval_runtime": 86.3257, + "eval_samples_per_second": 15.928, + "eval_steps_per_second": 1.992, + "step": 580 + }, + { + "entropy": 0.5822233572602272, + "epoch": 1.4931506849315068, + "grad_norm": 0.832811176776886, + "learning_rate": 0.0002246281760679571, + "loss": 0.5295282363891601, + "mean_token_accuracy": 0.8504064798355102, + "num_tokens": 1399537.0, + "step": 600 + }, + { + "epoch": 1.4931506849315068, + "eval_entropy": 0.5829724387027496, + "eval_loss": 0.5612193942070007, + "eval_mean_token_accuracy": 0.8449643853791925, + "eval_num_tokens": 1399537.0, + "eval_runtime": 86.6617, + "eval_samples_per_second": 15.866, + "eval_steps_per_second": 1.985, + "step": 600 + }, + { + "entropy": 0.571855777129531, + "epoch": 1.5429638854296388, + "grad_norm": 0.7665547728538513, + "learning_rate": 0.00022427691370553263, + "loss": 0.5187931060791016, + "mean_token_accuracy": 0.8534420043230057, + "num_tokens": 1448422.0, + "step": 620 + }, + { + "epoch": 1.5429638854296388, + "eval_entropy": 0.5623592240519302, + "eval_loss": 0.5575760006904602, + "eval_mean_token_accuracy": 0.8468210229346919, + "eval_num_tokens": 1448422.0, + "eval_runtime": 86.6324, + "eval_samples_per_second": 15.872, + "eval_steps_per_second": 1.985, + "step": 620 + }, + { + "entropy": 0.5740394659340382, + "epoch": 1.592777085927771, + "grad_norm": 0.6500429511070251, + "learning_rate": 0.00022389213388387174, + "loss": 0.5283198833465577, + "mean_token_accuracy": 0.8502798482775689, + "num_tokens": 1495009.0, + "step": 640 + }, + { + "epoch": 1.592777085927771, + "eval_entropy": 0.5548852207355721, + "eval_loss": 0.5561797022819519, + "eval_mean_token_accuracy": 0.8452786498291548, + "eval_num_tokens": 1495009.0, + "eval_runtime": 86.5205, + "eval_samples_per_second": 15.892, + "eval_steps_per_second": 1.988, + "step": 640 + }, + { + "entropy": 0.6020145989954472, + "epoch": 1.6425902864259028, + "grad_norm": 0.7056867480278015, + "learning_rate": 0.00022347395264732053, + "loss": 0.5400049209594726, + "mean_token_accuracy": 0.8447613954544068, + "num_tokens": 1536932.0, + "step": 660 + }, + { + "epoch": 1.6425902864259028, + "eval_entropy": 0.5618055154417836, + "eval_loss": 0.556106686592102, + "eval_mean_token_accuracy": 0.8465680112672407, + "eval_num_tokens": 1536932.0, + "eval_runtime": 86.2971, + "eval_samples_per_second": 15.933, + "eval_steps_per_second": 1.993, + "step": 660 + }, + { + "entropy": 0.5665927153080702, + "epoch": 1.692403486924035, + "grad_norm": 0.5987663865089417, + "learning_rate": 0.00022302249611363625, + "loss": 0.5143643856048584, + "mean_token_accuracy": 0.8529589556157589, + "num_tokens": 1585718.0, + "step": 680 + }, + { + "epoch": 1.692403486924035, + "eval_entropy": 0.568248552118623, + "eval_loss": 0.5476346015930176, + "eval_mean_token_accuracy": 0.8476775434128073, + "eval_num_tokens": 1585718.0, + "eval_runtime": 86.9583, + "eval_samples_per_second": 15.812, + "eval_steps_per_second": 1.978, + "step": 680 + }, + { + "entropy": 0.5673687808215618, + "epoch": 1.7422166874221667, + "grad_norm": 0.735261857509613, + "learning_rate": 0.00022253790043595193, + "loss": 0.509885597229004, + "mean_token_accuracy": 0.8537046857178211, + "num_tokens": 1635718.0, + "step": 700 + }, + { + "epoch": 1.7422166874221667, + "eval_entropy": 0.5616967284748721, + "eval_loss": 0.5439274311065674, + "eval_mean_token_accuracy": 0.8488946217437123, + "eval_num_tokens": 1635718.0, + "eval_runtime": 86.0604, + "eval_samples_per_second": 15.977, + "eval_steps_per_second": 1.999, + "step": 700 + }, + { + "entropy": 0.5529541682451964, + "epoch": 1.792029887920299, + "grad_norm": 0.7014835476875305, + "learning_rate": 0.00022202031176171442, + "loss": 0.5078992366790771, + "mean_token_accuracy": 0.8525233261287213, + "num_tokens": 1681291.0, + "step": 720 + }, + { + "epoch": 1.792029887920299, + "eval_entropy": 0.5827173320359962, + "eval_loss": 0.5419450402259827, + "eval_mean_token_accuracy": 0.8477318609176681, + "eval_num_tokens": 1681291.0, + "eval_runtime": 85.2984, + "eval_samples_per_second": 16.12, + "eval_steps_per_second": 2.016, + "step": 720 + }, + { + "entropy": 0.5755720350891351, + "epoch": 1.841843088418431, + "grad_norm": 0.705613911151886, + "learning_rate": 0.00022146988618860824, + "loss": 0.5181350708007812, + "mean_token_accuracy": 0.8467609457671642, + "num_tokens": 1729102.0, + "step": 740 + }, + { + "epoch": 1.841843088418431, + "eval_entropy": 0.5743971356125765, + "eval_loss": 0.5415896773338318, + "eval_mean_token_accuracy": 0.847328585940738, + "eval_num_tokens": 1729102.0, + "eval_runtime": 85.5602, + "eval_samples_per_second": 16.071, + "eval_steps_per_second": 2.01, + "step": 740 + }, + { + "entropy": 0.561330484598875, + "epoch": 1.891656288916563, + "grad_norm": 0.6722865700721741, + "learning_rate": 0.0002208867897174789, + "loss": 0.499837589263916, + "mean_token_accuracy": 0.8518734864890576, + "num_tokens": 1773578.0, + "step": 760 + }, + { + "epoch": 1.891656288916563, + "eval_entropy": 0.5865232653396074, + "eval_loss": 0.5437926650047302, + "eval_mean_token_accuracy": 0.8450997017843779, + "eval_num_tokens": 1773578.0, + "eval_runtime": 86.4116, + "eval_samples_per_second": 15.912, + "eval_steps_per_second": 1.99, + "step": 760 + }, + { + "entropy": 0.547389242425561, + "epoch": 1.9414694894146949, + "grad_norm": 0.7935577034950256, + "learning_rate": 0.00022027119820226907, + "loss": 0.4977591514587402, + "mean_token_accuracy": 0.8539491161704064, + "num_tokens": 1821725.0, + "step": 780 + }, + { + "epoch": 1.9414694894146949, + "eval_entropy": 0.5290903090391048, + "eval_loss": 0.5409526824951172, + "eval_mean_token_accuracy": 0.8497545698354411, + "eval_num_tokens": 1821725.0, + "eval_runtime": 86.7262, + "eval_samples_per_second": 15.854, + "eval_steps_per_second": 1.983, + "step": 780 + }, + { + "entropy": 0.5687909748405218, + "epoch": 1.9912826899128269, + "grad_norm": 0.6180546283721924, + "learning_rate": 0.00021962329729698345, + "loss": 0.5109643459320068, + "mean_token_accuracy": 0.8521598495543004, + "num_tokens": 1868431.0, + "step": 800 + }, + { + "epoch": 1.9912826899128269, + "eval_entropy": 0.5503541858390321, + "eval_loss": 0.5361555218696594, + "eval_mean_token_accuracy": 0.8510884285666221, + "eval_num_tokens": 1868431.0, + "eval_runtime": 86.3339, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 800 + }, + { + "entropy": 0.4739728841261986, + "epoch": 2.0398505603985058, + "grad_norm": 0.8058829307556152, + "learning_rate": 0.0002189432823996982, + "loss": 0.4204097747802734, + "mean_token_accuracy": 0.8728981889211215, + "num_tokens": 1915280.0, + "step": 820 + }, + { + "epoch": 2.0398505603985058, + "eval_entropy": 0.5077334992414297, + "eval_loss": 0.5531114339828491, + "eval_mean_token_accuracy": 0.8489257208136625, + "eval_num_tokens": 1915280.0, + "eval_runtime": 86.4801, + "eval_samples_per_second": 15.9, + "eval_steps_per_second": 1.989, + "step": 820 + }, + { + "entropy": 0.4594309840351343, + "epoch": 2.0896637608966375, + "grad_norm": 0.6906896829605103, + "learning_rate": 0.0002182313585936314, + "loss": 0.4071959495544434, + "mean_token_accuracy": 0.8732857562601566, + "num_tokens": 1965306.0, + "step": 840 + }, + { + "epoch": 2.0896637608966375, + "eval_entropy": 0.49850136994622474, + "eval_loss": 0.5486204624176025, + "eval_mean_token_accuracy": 0.8507991450470548, + "eval_num_tokens": 1965306.0, + "eval_runtime": 86.3364, + "eval_samples_per_second": 15.926, + "eval_steps_per_second": 1.992, + "step": 840 + }, + { + "entropy": 0.4881629109382629, + "epoch": 2.1394769613947697, + "grad_norm": 0.6343470215797424, + "learning_rate": 0.0002174877405852928, + "loss": 0.41669540405273436, + "mean_token_accuracy": 0.8711295068264008, + "num_tokens": 2008562.0, + "step": 860 + }, + { + "epoch": 2.1394769613947697, + "eval_entropy": 0.49155513924914734, + "eval_loss": 0.555109441280365, + "eval_mean_token_accuracy": 0.8496399400539176, + "eval_num_tokens": 2008562.0, + "eval_runtime": 86.3295, + "eval_samples_per_second": 15.927, + "eval_steps_per_second": 1.992, + "step": 860 + }, + { + "entropy": 0.4648668970912695, + "epoch": 2.1892901618929015, + "grad_norm": 0.8014165163040161, + "learning_rate": 0.00021671265263973133, + "loss": 0.4110250473022461, + "mean_token_accuracy": 0.8754166305065155, + "num_tokens": 2056474.0, + "step": 880 + }, + { + "epoch": 2.1892901618929015, + "eval_entropy": 0.4909258722219356, + "eval_loss": 0.5539511442184448, + "eval_mean_token_accuracy": 0.8492401502160138, + "eval_num_tokens": 2056474.0, + "eval_runtime": 86.3468, + "eval_samples_per_second": 15.924, + "eval_steps_per_second": 1.992, + "step": 880 + }, + { + "entropy": 0.4824485514312983, + "epoch": 2.2391033623910337, + "grad_norm": 0.6665191054344177, + "learning_rate": 0.00021590632851289967, + "loss": 0.4181404113769531, + "mean_token_accuracy": 0.8726993151009083, + "num_tokens": 2103543.0, + "step": 900 + }, + { + "epoch": 2.2391033623910337, + "eval_entropy": 0.4986876940657926, + "eval_loss": 0.547695517539978, + "eval_mean_token_accuracy": 0.8501384708770486, + "eval_num_tokens": 2103543.0, + "eval_runtime": 86.3838, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 900 + }, + { + "entropy": 0.4751896943897009, + "epoch": 2.2889165628891655, + "grad_norm": 0.81158047914505, + "learning_rate": 0.00021506901138115678, + "loss": 0.40689678192138673, + "mean_token_accuracy": 0.8745221219956875, + "num_tokens": 2147861.0, + "step": 920 + }, + { + "epoch": 2.2889165628891655, + "eval_entropy": 0.507153491121392, + "eval_loss": 0.5501641631126404, + "eval_mean_token_accuracy": 0.8495670116918032, + "eval_num_tokens": 2147861.0, + "eval_runtime": 86.0912, + "eval_samples_per_second": 15.971, + "eval_steps_per_second": 1.998, + "step": 920 + }, + { + "entropy": 0.4873133715242147, + "epoch": 2.3387297633872977, + "grad_norm": 0.7218056321144104, + "learning_rate": 0.0002142009537679292, + "loss": 0.42701358795166017, + "mean_token_accuracy": 0.8695114746689796, + "num_tokens": 2190561.0, + "step": 940 + }, + { + "epoch": 2.3387297633872977, + "eval_entropy": 0.5202612736543943, + "eval_loss": 0.5491839051246643, + "eval_mean_token_accuracy": 0.8494071208460386, + "eval_num_tokens": 2190561.0, + "eval_runtime": 86.1142, + "eval_samples_per_second": 15.967, + "eval_steps_per_second": 1.997, + "step": 940 + }, + { + "entropy": 0.4762951169162989, + "epoch": 2.3885429638854294, + "grad_norm": 0.7194424867630005, + "learning_rate": 0.0002133024174675534, + "loss": 0.42299847602844237, + "mean_token_accuracy": 0.8709790132939815, + "num_tokens": 2239412.0, + "step": 960 + }, + { + "epoch": 2.3885429638854294, + "eval_entropy": 0.4899340462546016, + "eval_loss": 0.5522511601448059, + "eval_mean_token_accuracy": 0.8492208258357159, + "eval_num_tokens": 2239412.0, + "eval_runtime": 86.463, + "eval_samples_per_second": 15.903, + "eval_steps_per_second": 1.989, + "step": 960 + }, + { + "entropy": 0.49650347977876663, + "epoch": 2.4383561643835616, + "grad_norm": 0.8406022787094116, + "learning_rate": 0.0002123736734663221, + "loss": 0.4275330066680908, + "mean_token_accuracy": 0.8670595556497573, + "num_tokens": 2286283.0, + "step": 980 + }, + { + "epoch": 2.4383561643835616, + "eval_entropy": 0.49691385654515996, + "eval_loss": 0.5491269826889038, + "eval_mean_token_accuracy": 0.850309816210769, + "eval_num_tokens": 2286283.0, + "eval_runtime": 86.17, + "eval_samples_per_second": 15.957, + "eval_steps_per_second": 1.996, + "step": 980 + }, + { + "entropy": 0.48843890577554705, + "epoch": 2.488169364881694, + "grad_norm": 0.9082473516464233, + "learning_rate": 0.00021141500186075868, + "loss": 0.4309722423553467, + "mean_token_accuracy": 0.8686766296625137, + "num_tokens": 2333733.0, + "step": 1000 + }, + { + "epoch": 2.488169364881694, + "eval_entropy": 0.5543508351195691, + "eval_loss": 0.5478800535202026, + "eval_mean_token_accuracy": 0.8478029522784921, + "eval_num_tokens": 2333733.0, + "eval_runtime": 86.3835, + "eval_samples_per_second": 15.917, + "eval_steps_per_second": 1.991, + "step": 1000 + }, + { + "entropy": 0.4777219031006098, + "epoch": 2.5379825653798256, + "grad_norm": 0.7448089122772217, + "learning_rate": 0.0002104266917731438, + "loss": 0.423325252532959, + "mean_token_accuracy": 0.8706337086856365, + "num_tokens": 2384270.0, + "step": 1020 + }, + { + "epoch": 2.5379825653798256, + "eval_entropy": 0.49857561550168106, + "eval_loss": 0.5511948466300964, + "eval_mean_token_accuracy": 0.8502220289651737, + "eval_num_tokens": 2384270.0, + "eval_runtime": 86.5399, + "eval_samples_per_second": 15.889, + "eval_steps_per_second": 1.988, + "step": 1020 + }, + { + "entropy": 0.4844174191355705, + "epoch": 2.587795765877958, + "grad_norm": 0.794029176235199, + "learning_rate": 0.00020940904126432, + "loss": 0.4176753044128418, + "mean_token_accuracy": 0.873535567522049, + "num_tokens": 2428036.0, + "step": 1040 + }, + { + "epoch": 2.587795765877958, + "eval_entropy": 0.485467542222766, + "eval_loss": 0.5539286732673645, + "eval_mean_token_accuracy": 0.8495475081510322, + "eval_num_tokens": 2428036.0, + "eval_runtime": 86.135, + "eval_samples_per_second": 15.963, + "eval_steps_per_second": 1.997, + "step": 1040 + }, + { + "entropy": 0.49070929251611234, + "epoch": 2.6376089663760895, + "grad_norm": 0.7558256983757019, + "learning_rate": 0.0002083623572438007, + "loss": 0.42867293357849123, + "mean_token_accuracy": 0.8696666076779366, + "num_tokens": 2476815.0, + "step": 1060 + }, + { + "epoch": 2.6376089663760895, + "eval_entropy": 0.490822730889154, + "eval_loss": 0.5434785485267639, + "eval_mean_token_accuracy": 0.850568296950917, + "eval_num_tokens": 2476815.0, + "eval_runtime": 86.4933, + "eval_samples_per_second": 15.897, + "eval_steps_per_second": 1.989, + "step": 1060 + }, + { + "entropy": 0.47806114703416824, + "epoch": 2.6874221668742218, + "grad_norm": 0.6608979105949402, + "learning_rate": 0.00020728695537721047, + "loss": 0.4289727687835693, + "mean_token_accuracy": 0.8693130135536193, + "num_tokens": 2527131.0, + "step": 1080 + }, + { + "epoch": 2.6874221668742218, + "eval_entropy": 0.5285773256490397, + "eval_loss": 0.5444230437278748, + "eval_mean_token_accuracy": 0.8498796481032704, + "eval_num_tokens": 2527131.0, + "eval_runtime": 86.7091, + "eval_samples_per_second": 15.858, + "eval_steps_per_second": 1.984, + "step": 1080 + } + ], + "logging_steps": 20, + "max_steps": 4020, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0682640451304448e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/README.md b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/adapter_config.json b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4e1911440bbdf4e34c1f99d492253e23cdedc8b2 --- /dev/null +++ b/overgeneralisation_original_Swedish/Qwen3.5-4B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-1100/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.028265386974777595, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "up_proj", + "q_proj", + "o_proj", + "v_proj", + "k_proj", + "gate_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f65a172be206bf647f3c77c02eb48e060c3723b9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/5khkq4dz) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c3791c00110779dd96c425d3114da3ddd6e16fe3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1266/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1266, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.244685217283277e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d11c32891287f73784d84b55597305f80dc370ba --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1688/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1688, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.33006276254679e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..592e3293d2d617f3e56ca779fa129fd57640367e --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2110/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2110, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0419291201852211e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..240053cbe2c7358bcd1a8d10c66ce7a7bcd2e152 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2532/trainer_state.json @@ -0,0 +1,600 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2532, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + }, + { + "entropy": 0.38868104307179524, + "epoch": 5.094955489614243, + "grad_norm": 1.2374166250228882, + "learning_rate": 4.697173902129824e-05, + "loss": 0.2337013626098633, + "mean_token_accuracy": 0.9221220642782105, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.3672105521708727, + "epoch": 5.213649851632048, + "grad_norm": 1.5339794158935547, + "learning_rate": 4.528299262330934e-05, + "loss": 0.21680095672607422, + "mean_token_accuracy": 0.9280403655767441, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3635194416344166, + "epoch": 5.332344213649852, + "grad_norm": 1.6350411176681519, + "learning_rate": 4.3587158053244715e-05, + "loss": 0.2198893928527832, + "mean_token_accuracy": 0.9272216448187828, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.36030806206166743, + "epoch": 5.451038575667655, + "grad_norm": 1.4245432615280151, + "learning_rate": 4.188713566778936e-05, + "loss": 0.21613813400268556, + "mean_token_accuracy": 0.9288999038934708, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.3618554932624102, + "epoch": 5.56973293768546, + "grad_norm": 1.609066367149353, + "learning_rate": 4.0185832985976856e-05, + "loss": 0.21525728225708007, + "mean_token_accuracy": 0.9276236236095429, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3609679951518774, + "epoch": 5.688427299703264, + "grad_norm": 1.0492284297943115, + "learning_rate": 3.848615971650962e-05, + "loss": 0.2146327590942383, + "mean_token_accuracy": 0.9283027359843254, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.34350181475281716, + "epoch": 5.807121661721069, + "grad_norm": 1.6522330045700073, + "learning_rate": 3.679102278133421e-05, + "loss": 0.20981479644775392, + "mean_token_accuracy": 0.9296323484182358, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.3616155734658241, + "epoch": 5.925816023738872, + "grad_norm": 1.762353777885437, + "learning_rate": 3.5103321343982645e-05, + "loss": 0.22292245864868165, + "mean_token_accuracy": 0.9254147034883499, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3478013425194032, + "eval_loss": 0.8838939666748047, + "eval_mean_token_accuracy": 0.8098702326155546, + "eval_num_tokens": 3357462.0, + "eval_runtime": 111.8466, + "eval_samples_per_second": 12.195, + "eval_steps_per_second": 1.529, + "step": 2532 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2493870454255616e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..04908bf059dbef671c81fce2ee8d68262a31eb50 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2954/trainer_state.json @@ -0,0 +1,701 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2954, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + }, + { + "entropy": 0.38868104307179524, + "epoch": 5.094955489614243, + "grad_norm": 1.2374166250228882, + "learning_rate": 4.697173902129824e-05, + "loss": 0.2337013626098633, + "mean_token_accuracy": 0.9221220642782105, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.3672105521708727, + "epoch": 5.213649851632048, + "grad_norm": 1.5339794158935547, + "learning_rate": 4.528299262330934e-05, + "loss": 0.21680095672607422, + "mean_token_accuracy": 0.9280403655767441, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3635194416344166, + "epoch": 5.332344213649852, + "grad_norm": 1.6350411176681519, + "learning_rate": 4.3587158053244715e-05, + "loss": 0.2198893928527832, + "mean_token_accuracy": 0.9272216448187828, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.36030806206166743, + "epoch": 5.451038575667655, + "grad_norm": 1.4245432615280151, + "learning_rate": 4.188713566778936e-05, + "loss": 0.21613813400268556, + "mean_token_accuracy": 0.9288999038934708, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.3618554932624102, + "epoch": 5.56973293768546, + "grad_norm": 1.609066367149353, + "learning_rate": 4.0185832985976856e-05, + "loss": 0.21525728225708007, + "mean_token_accuracy": 0.9276236236095429, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3609679951518774, + "epoch": 5.688427299703264, + "grad_norm": 1.0492284297943115, + "learning_rate": 3.848615971650962e-05, + "loss": 0.2146327590942383, + "mean_token_accuracy": 0.9283027359843254, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.34350181475281716, + "epoch": 5.807121661721069, + "grad_norm": 1.6522330045700073, + "learning_rate": 3.679102278133421e-05, + "loss": 0.20981479644775392, + "mean_token_accuracy": 0.9296323484182358, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.3616155734658241, + "epoch": 5.925816023738872, + "grad_norm": 1.762353777885437, + "learning_rate": 3.5103321343982645e-05, + "loss": 0.22292245864868165, + "mean_token_accuracy": 0.9254147034883499, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3478013425194032, + "eval_loss": 0.8838939666748047, + "eval_mean_token_accuracy": 0.8098702326155546, + "eval_num_tokens": 3357462.0, + "eval_runtime": 111.8466, + "eval_samples_per_second": 12.195, + "eval_steps_per_second": 1.529, + "step": 2532 + }, + { + "entropy": 0.33996166356929064, + "epoch": 6.04272997032641, + "grad_norm": 1.509031891822815, + "learning_rate": 3.342594185118299e-05, + "loss": 0.19556612014770508, + "mean_token_accuracy": 0.9348860970003351, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.29004726007580756, + "epoch": 6.161424332344214, + "grad_norm": 1.3547477722167969, + "learning_rate": 3.176175309621916e-05, + "loss": 0.1560215663909912, + "mean_token_accuracy": 0.9475028210878372, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.2834444552659988, + "epoch": 6.2801186943620175, + "grad_norm": 1.6714046001434326, + "learning_rate": 3.01136013124831e-05, + "loss": 0.15274467468261718, + "mean_token_accuracy": 0.9488504147529602, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.30035923302173617, + "epoch": 6.398813056379822, + "grad_norm": 1.149774432182312, + "learning_rate": 2.8484305305610797e-05, + "loss": 0.16209911346435546, + "mean_token_accuracy": 0.9469243207573891, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.28467367485165596, + "epoch": 6.517507418397626, + "grad_norm": 1.2987172603607178, + "learning_rate": 2.6876651632527578e-05, + "loss": 0.15439175605773925, + "mean_token_accuracy": 0.9486008253693581, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2905779552459717, + "epoch": 6.63620178041543, + "grad_norm": 1.3652619123458862, + "learning_rate": 2.529338983564765e-05, + "loss": 0.15736764907836914, + "mean_token_accuracy": 0.9474438297748565, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.27992172554135325, + "epoch": 6.7548961424332346, + "grad_norm": 1.092677354812622, + "learning_rate": 2.3737227740379025e-05, + "loss": 0.15728514671325683, + "mean_token_accuracy": 0.9473575839400291, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.28447474762797353, + "epoch": 6.873590504451038, + "grad_norm": 1.409013271331787, + "learning_rate": 2.221082682397646e-05, + "loss": 0.15529794692993165, + "mean_token_accuracy": 0.9479124927520752, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.2843698911368847, + "epoch": 6.992284866468843, + "grad_norm": 0.948100209236145, + "learning_rate": 2.0716797663662474e-05, + "loss": 0.15718685150146483, + "mean_token_accuracy": 0.9478301125764846, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29428490229517396, + "eval_loss": 1.0146509408950806, + "eval_mean_token_accuracy": 0.8090401922750194, + "eval_num_tokens": 3917039.0, + "eval_runtime": 112.5894, + "eval_samples_per_second": 12.115, + "eval_steps_per_second": 1.519, + "step": 2954 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4583844510858445e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..291eb1e79732e3b782fdbf33720a78388d141f51 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3376/trainer_state.json @@ -0,0 +1,792 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3376, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + }, + { + "entropy": 0.38868104307179524, + "epoch": 5.094955489614243, + "grad_norm": 1.2374166250228882, + "learning_rate": 4.697173902129824e-05, + "loss": 0.2337013626098633, + "mean_token_accuracy": 0.9221220642782105, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.3672105521708727, + "epoch": 5.213649851632048, + "grad_norm": 1.5339794158935547, + "learning_rate": 4.528299262330934e-05, + "loss": 0.21680095672607422, + "mean_token_accuracy": 0.9280403655767441, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3635194416344166, + "epoch": 5.332344213649852, + "grad_norm": 1.6350411176681519, + "learning_rate": 4.3587158053244715e-05, + "loss": 0.2198893928527832, + "mean_token_accuracy": 0.9272216448187828, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.36030806206166743, + "epoch": 5.451038575667655, + "grad_norm": 1.4245432615280151, + "learning_rate": 4.188713566778936e-05, + "loss": 0.21613813400268556, + "mean_token_accuracy": 0.9288999038934708, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.3618554932624102, + "epoch": 5.56973293768546, + "grad_norm": 1.609066367149353, + "learning_rate": 4.0185832985976856e-05, + "loss": 0.21525728225708007, + "mean_token_accuracy": 0.9276236236095429, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3609679951518774, + "epoch": 5.688427299703264, + "grad_norm": 1.0492284297943115, + "learning_rate": 3.848615971650962e-05, + "loss": 0.2146327590942383, + "mean_token_accuracy": 0.9283027359843254, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.34350181475281716, + "epoch": 5.807121661721069, + "grad_norm": 1.6522330045700073, + "learning_rate": 3.679102278133421e-05, + "loss": 0.20981479644775392, + "mean_token_accuracy": 0.9296323484182358, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.3616155734658241, + "epoch": 5.925816023738872, + "grad_norm": 1.762353777885437, + "learning_rate": 3.5103321343982645e-05, + "loss": 0.22292245864868165, + "mean_token_accuracy": 0.9254147034883499, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3478013425194032, + "eval_loss": 0.8838939666748047, + "eval_mean_token_accuracy": 0.8098702326155546, + "eval_num_tokens": 3357462.0, + "eval_runtime": 111.8466, + "eval_samples_per_second": 12.195, + "eval_steps_per_second": 1.529, + "step": 2532 + }, + { + "entropy": 0.33996166356929064, + "epoch": 6.04272997032641, + "grad_norm": 1.509031891822815, + "learning_rate": 3.342594185118299e-05, + "loss": 0.19556612014770508, + "mean_token_accuracy": 0.9348860970003351, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.29004726007580756, + "epoch": 6.161424332344214, + "grad_norm": 1.3547477722167969, + "learning_rate": 3.176175309621916e-05, + "loss": 0.1560215663909912, + "mean_token_accuracy": 0.9475028210878372, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.2834444552659988, + "epoch": 6.2801186943620175, + "grad_norm": 1.6714046001434326, + "learning_rate": 3.01136013124831e-05, + "loss": 0.15274467468261718, + "mean_token_accuracy": 0.9488504147529602, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.30035923302173617, + "epoch": 6.398813056379822, + "grad_norm": 1.149774432182312, + "learning_rate": 2.8484305305610797e-05, + "loss": 0.16209911346435546, + "mean_token_accuracy": 0.9469243207573891, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.28467367485165596, + "epoch": 6.517507418397626, + "grad_norm": 1.2987172603607178, + "learning_rate": 2.6876651632527578e-05, + "loss": 0.15439175605773925, + "mean_token_accuracy": 0.9486008253693581, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2905779552459717, + "epoch": 6.63620178041543, + "grad_norm": 1.3652619123458862, + "learning_rate": 2.529338983564765e-05, + "loss": 0.15736764907836914, + "mean_token_accuracy": 0.9474438297748565, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.27992172554135325, + "epoch": 6.7548961424332346, + "grad_norm": 1.092677354812622, + "learning_rate": 2.3737227740379025e-05, + "loss": 0.15728514671325683, + "mean_token_accuracy": 0.9473575839400291, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.28447474762797353, + "epoch": 6.873590504451038, + "grad_norm": 1.409013271331787, + "learning_rate": 2.221082682397646e-05, + "loss": 0.15529794692993165, + "mean_token_accuracy": 0.9479124927520752, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.2843698911368847, + "epoch": 6.992284866468843, + "grad_norm": 0.948100209236145, + "learning_rate": 2.0716797663662474e-05, + "loss": 0.15718685150146483, + "mean_token_accuracy": 0.9478301125764846, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29428490229517396, + "eval_loss": 1.0146509408950806, + "eval_mean_token_accuracy": 0.8090401922750194, + "eval_num_tokens": 3917039.0, + "eval_runtime": 112.5894, + "eval_samples_per_second": 12.115, + "eval_steps_per_second": 1.519, + "step": 2954 + }, + { + "entropy": 0.26112703653761576, + "epoch": 7.10919881305638, + "grad_norm": 1.360489010810852, + "learning_rate": 1.9257695471802237e-05, + "loss": 0.1267384433746338, + "mean_token_accuracy": 0.957306801970235, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.2523047244548798, + "epoch": 7.227893175074184, + "grad_norm": 0.7781679630279541, + "learning_rate": 1.783601572576776e-05, + "loss": 0.12492096900939942, + "mean_token_accuracy": 0.9576244935393333, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.2563566393405199, + "epoch": 7.346587537091988, + "grad_norm": 1.2237826585769653, + "learning_rate": 1.645418989996586e-05, + "loss": 0.12833234786987305, + "mean_token_accuracy": 0.9578728675842285, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.24430972374975682, + "epoch": 7.465281899109792, + "grad_norm": 1.0145419836044312, + "learning_rate": 1.5114581307329361e-05, + "loss": 0.1226680850982666, + "mean_token_accuracy": 0.9584381604194641, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.2510662242770195, + "epoch": 7.583976261127597, + "grad_norm": 0.8724110722541809, + "learning_rate": 1.3819481057383636e-05, + "loss": 0.12699291229248047, + "mean_token_accuracy": 0.9558046141266823, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.2556200893968344, + "epoch": 7.7026706231454005, + "grad_norm": 1.4136648178100586, + "learning_rate": 1.2571104137801403e-05, + "loss": 0.13246843338012695, + "mean_token_accuracy": 0.9558334472775459, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.250102458037436, + "epoch": 7.821364985163205, + "grad_norm": 0.8711676597595215, + "learning_rate": 1.1371585626147424e-05, + "loss": 0.12776278495788573, + "mean_token_accuracy": 0.9560130417346955, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.2509234543889761, + "epoch": 7.940059347181009, + "grad_norm": 0.7930846214294434, + "learning_rate": 1.0222977038292094e-05, + "loss": 0.12907642364501953, + "mean_token_accuracy": 0.9563833594322204, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2663285747083307, + "eval_loss": 1.1223618984222412, + "eval_mean_token_accuracy": 0.8081985771307472, + "eval_num_tokens": 4476616.0, + "eval_runtime": 111.693, + "eval_samples_per_second": 12.212, + "eval_steps_per_second": 1.531, + "step": 3376 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6655002695661978e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6554fbb9c9b54092ca0c0c936643c9cc9c6c6d35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3798/trainer_state.json @@ -0,0 +1,883 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3798, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + }, + { + "entropy": 0.38868104307179524, + "epoch": 5.094955489614243, + "grad_norm": 1.2374166250228882, + "learning_rate": 4.697173902129824e-05, + "loss": 0.2337013626098633, + "mean_token_accuracy": 0.9221220642782105, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.3672105521708727, + "epoch": 5.213649851632048, + "grad_norm": 1.5339794158935547, + "learning_rate": 4.528299262330934e-05, + "loss": 0.21680095672607422, + "mean_token_accuracy": 0.9280403655767441, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3635194416344166, + "epoch": 5.332344213649852, + "grad_norm": 1.6350411176681519, + "learning_rate": 4.3587158053244715e-05, + "loss": 0.2198893928527832, + "mean_token_accuracy": 0.9272216448187828, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.36030806206166743, + "epoch": 5.451038575667655, + "grad_norm": 1.4245432615280151, + "learning_rate": 4.188713566778936e-05, + "loss": 0.21613813400268556, + "mean_token_accuracy": 0.9288999038934708, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.3618554932624102, + "epoch": 5.56973293768546, + "grad_norm": 1.609066367149353, + "learning_rate": 4.0185832985976856e-05, + "loss": 0.21525728225708007, + "mean_token_accuracy": 0.9276236236095429, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3609679951518774, + "epoch": 5.688427299703264, + "grad_norm": 1.0492284297943115, + "learning_rate": 3.848615971650962e-05, + "loss": 0.2146327590942383, + "mean_token_accuracy": 0.9283027359843254, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.34350181475281716, + "epoch": 5.807121661721069, + "grad_norm": 1.6522330045700073, + "learning_rate": 3.679102278133421e-05, + "loss": 0.20981479644775392, + "mean_token_accuracy": 0.9296323484182358, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.3616155734658241, + "epoch": 5.925816023738872, + "grad_norm": 1.762353777885437, + "learning_rate": 3.5103321343982645e-05, + "loss": 0.22292245864868165, + "mean_token_accuracy": 0.9254147034883499, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3478013425194032, + "eval_loss": 0.8838939666748047, + "eval_mean_token_accuracy": 0.8098702326155546, + "eval_num_tokens": 3357462.0, + "eval_runtime": 111.8466, + "eval_samples_per_second": 12.195, + "eval_steps_per_second": 1.529, + "step": 2532 + }, + { + "entropy": 0.33996166356929064, + "epoch": 6.04272997032641, + "grad_norm": 1.509031891822815, + "learning_rate": 3.342594185118299e-05, + "loss": 0.19556612014770508, + "mean_token_accuracy": 0.9348860970003351, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.29004726007580756, + "epoch": 6.161424332344214, + "grad_norm": 1.3547477722167969, + "learning_rate": 3.176175309621916e-05, + "loss": 0.1560215663909912, + "mean_token_accuracy": 0.9475028210878372, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.2834444552659988, + "epoch": 6.2801186943620175, + "grad_norm": 1.6714046001434326, + "learning_rate": 3.01136013124831e-05, + "loss": 0.15274467468261718, + "mean_token_accuracy": 0.9488504147529602, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.30035923302173617, + "epoch": 6.398813056379822, + "grad_norm": 1.149774432182312, + "learning_rate": 2.8484305305610797e-05, + "loss": 0.16209911346435546, + "mean_token_accuracy": 0.9469243207573891, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.28467367485165596, + "epoch": 6.517507418397626, + "grad_norm": 1.2987172603607178, + "learning_rate": 2.6876651632527578e-05, + "loss": 0.15439175605773925, + "mean_token_accuracy": 0.9486008253693581, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2905779552459717, + "epoch": 6.63620178041543, + "grad_norm": 1.3652619123458862, + "learning_rate": 2.529338983564765e-05, + "loss": 0.15736764907836914, + "mean_token_accuracy": 0.9474438297748565, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.27992172554135325, + "epoch": 6.7548961424332346, + "grad_norm": 1.092677354812622, + "learning_rate": 2.3737227740379025e-05, + "loss": 0.15728514671325683, + "mean_token_accuracy": 0.9473575839400291, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.28447474762797353, + "epoch": 6.873590504451038, + "grad_norm": 1.409013271331787, + "learning_rate": 2.221082682397646e-05, + "loss": 0.15529794692993165, + "mean_token_accuracy": 0.9479124927520752, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.2843698911368847, + "epoch": 6.992284866468843, + "grad_norm": 0.948100209236145, + "learning_rate": 2.0716797663662474e-05, + "loss": 0.15718685150146483, + "mean_token_accuracy": 0.9478301125764846, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29428490229517396, + "eval_loss": 1.0146509408950806, + "eval_mean_token_accuracy": 0.8090401922750194, + "eval_num_tokens": 3917039.0, + "eval_runtime": 112.5894, + "eval_samples_per_second": 12.115, + "eval_steps_per_second": 1.519, + "step": 2954 + }, + { + "entropy": 0.26112703653761576, + "epoch": 7.10919881305638, + "grad_norm": 1.360489010810852, + "learning_rate": 1.9257695471802237e-05, + "loss": 0.1267384433746338, + "mean_token_accuracy": 0.957306801970235, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.2523047244548798, + "epoch": 7.227893175074184, + "grad_norm": 0.7781679630279541, + "learning_rate": 1.783601572576776e-05, + "loss": 0.12492096900939942, + "mean_token_accuracy": 0.9576244935393333, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.2563566393405199, + "epoch": 7.346587537091988, + "grad_norm": 1.2237826585769653, + "learning_rate": 1.645418989996586e-05, + "loss": 0.12833234786987305, + "mean_token_accuracy": 0.9578728675842285, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.24430972374975682, + "epoch": 7.465281899109792, + "grad_norm": 1.0145419836044312, + "learning_rate": 1.5114581307329361e-05, + "loss": 0.1226680850982666, + "mean_token_accuracy": 0.9584381604194641, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.2510662242770195, + "epoch": 7.583976261127597, + "grad_norm": 0.8724110722541809, + "learning_rate": 1.3819481057383636e-05, + "loss": 0.12699291229248047, + "mean_token_accuracy": 0.9558046141266823, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.2556200893968344, + "epoch": 7.7026706231454005, + "grad_norm": 1.4136648178100586, + "learning_rate": 1.2571104137801403e-05, + "loss": 0.13246843338012695, + "mean_token_accuracy": 0.9558334472775459, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.250102458037436, + "epoch": 7.821364985163205, + "grad_norm": 0.8711676597595215, + "learning_rate": 1.1371585626147424e-05, + "loss": 0.12776278495788573, + "mean_token_accuracy": 0.9560130417346955, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.2509234543889761, + "epoch": 7.940059347181009, + "grad_norm": 0.7930846214294434, + "learning_rate": 1.0222977038292094e-05, + "loss": 0.12907642364501953, + "mean_token_accuracy": 0.9563833594322204, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2663285747083307, + "eval_loss": 1.1223618984222412, + "eval_mean_token_accuracy": 0.8081985771307472, + "eval_num_tokens": 4476616.0, + "eval_runtime": 111.693, + "eval_samples_per_second": 12.212, + "eval_steps_per_second": 1.531, + "step": 3376 + }, + { + "entropy": 0.2438919508290775, + "epoch": 8.056973293768547, + "grad_norm": 0.5826248526573181, + "learning_rate": 9.12724281973914e-06, + "loss": 0.11967051506042481, + "mean_token_accuracy": 0.9597063037344646, + "num_tokens": 4510235.0, + "step": 3400 + }, + { + "entropy": 0.22686645958572627, + "epoch": 8.17566765578635, + "grad_norm": 0.5468573570251465, + "learning_rate": 8.086256985868273e-06, + "loss": 0.10963334083557129, + "mean_token_accuracy": 0.9632616007328033, + "num_tokens": 4577780.0, + "step": 3450 + }, + { + "entropy": 0.2430728640407324, + "epoch": 8.294362017804154, + "grad_norm": 0.6237483620643616, + "learning_rate": 7.101799916838946e-06, + "loss": 0.11639170646667481, + "mean_token_accuracy": 0.9603625151515007, + "num_tokens": 4641616.0, + "step": 3500 + }, + { + "entropy": 0.24073882170021535, + "epoch": 8.413056379821958, + "grad_norm": 0.6401548981666565, + "learning_rate": 6.175555312636793e-06, + "loss": 0.11829930305480957, + "mean_token_accuracy": 0.9581007561087609, + "num_tokens": 4706065.0, + "step": 3550 + }, + { + "entropy": 0.22724292811006308, + "epoch": 8.531750741839762, + "grad_norm": 0.45143744349479675, + "learning_rate": 5.309107313470484e-06, + "loss": 0.11297051429748535, + "mean_token_accuracy": 0.9609669655561447, + "num_tokens": 4772757.0, + "step": 3600 + }, + { + "entropy": 0.2373272515460849, + "epoch": 8.650445103857567, + "grad_norm": 0.4666413962841034, + "learning_rate": 4.5039377904440675e-06, + "loss": 0.11534086227416993, + "mean_token_accuracy": 0.9599428379535675, + "num_tokens": 4839113.0, + "step": 3650 + }, + { + "entropy": 0.23692427456378937, + "epoch": 8.769139465875371, + "grad_norm": 0.581846296787262, + "learning_rate": 3.7614238111382377e-06, + "loss": 0.11592741012573242, + "mean_token_accuracy": 0.9601775661110878, + "num_tokens": 4904798.0, + "step": 3700 + }, + { + "entropy": 0.2240232530236244, + "epoch": 8.887833827893175, + "grad_norm": 0.5457125902175903, + "learning_rate": 3.08283528443536e-06, + "loss": 0.1130020809173584, + "mean_token_accuracy": 0.9610978499054909, + "num_tokens": 4972291.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24879027266948545, + "eval_loss": 1.2201728820800781, + "eval_mean_token_accuracy": 0.8093260432544508, + "eval_num_tokens": 5036193.0, + "eval_runtime": 112.3651, + "eval_samples_per_second": 12.139, + "eval_steps_per_second": 1.522, + "step": 3798 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.875156450038661e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8552b373a7bfa030f855aa081623127bb05d3475 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-422/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 422, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.0866582526214144e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..78e183767a59ec2df92b7ba10ae6624b809b4869 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4220/trainer_state.json @@ -0,0 +1,984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + }, + { + "entropy": 0.7529444349598764, + "epoch": 2.0142433234421366, + "grad_norm": 0.9705535173416138, + "learning_rate": 7.973761550607747e-05, + "loss": 0.6287842178344727, + "mean_token_accuracy": 0.8173542984851121, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6624361242353917, + "epoch": 2.1329376854599404, + "grad_norm": 1.0336796045303345, + "learning_rate": 7.911624507384729e-05, + "loss": 0.5305244064331055, + "mean_token_accuracy": 0.8395491230487824, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6706090711057187, + "epoch": 2.2516320474777447, + "grad_norm": 1.1563575267791748, + "learning_rate": 7.842992204004328e-05, + "loss": 0.5347850036621093, + "mean_token_accuracy": 0.8390352365374565, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6641162340342999, + "epoch": 2.370326409495549, + "grad_norm": 1.0999572277069092, + "learning_rate": 7.767982021114064e-05, + "loss": 0.5343616867065429, + "mean_token_accuracy": 0.8390876743197441, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6628478536009789, + "epoch": 2.489020771513353, + "grad_norm": 1.0276364088058472, + "learning_rate": 7.68672224733903e-05, + "loss": 0.5415428161621094, + "mean_token_accuracy": 0.8387553268671035, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.671518052071333, + "epoch": 2.6077151335311575, + "grad_norm": 0.9451322555541992, + "learning_rate": 7.599351859872084e-05, + "loss": 0.5410358810424805, + "mean_token_accuracy": 0.8373630735278129, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6752241159975528, + "epoch": 2.7264094955489613, + "grad_norm": 0.8325166702270508, + "learning_rate": 7.506020286783527e-05, + "loss": 0.5409298706054687, + "mean_token_accuracy": 0.8369611689448356, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6462954029440879, + "epoch": 2.8451038575667655, + "grad_norm": 1.0445443391799927, + "learning_rate": 7.406887151456858e-05, + "loss": 0.5271347427368164, + "mean_token_accuracy": 0.8404733729362488, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6678441441059113, + "epoch": 2.96379821958457, + "grad_norm": 0.9832372665405273, + "learning_rate": 7.302121999587646e-05, + "loss": 0.537381706237793, + "mean_token_accuracy": 0.8383750656247139, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.524203968675513, + "eval_loss": 0.6435813307762146, + "eval_mean_token_accuracy": 0.8187350073055915, + "eval_num_tokens": 1678731.0, + "eval_runtime": 111.7182, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1266 + }, + { + "entropy": 0.5911264679758682, + "epoch": 3.080712166172107, + "grad_norm": 1.2129673957824707, + "learning_rate": 7.19190400921244e-05, + "loss": 0.44908695220947265, + "mean_token_accuracy": 0.8601035639114186, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5566366592049599, + "epoch": 3.199406528189911, + "grad_norm": 0.9602940678596497, + "learning_rate": 7.076421684263661e-05, + "loss": 0.4135295867919922, + "mean_token_accuracy": 0.8689188846945762, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5605658321082592, + "epoch": 3.318100890207715, + "grad_norm": 1.0783617496490479, + "learning_rate": 6.955872532174566e-05, + "loss": 0.41924549102783204, + "mean_token_accuracy": 0.8669222807884216, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5531888791918754, + "epoch": 3.436795252225519, + "grad_norm": 1.285948395729065, + "learning_rate": 6.830462726085685e-05, + "loss": 0.41391544342041015, + "mean_token_accuracy": 0.8701067119836807, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5541697943210602, + "epoch": 3.5554896142433234, + "grad_norm": 1.3446345329284668, + "learning_rate": 6.700406752230453e-05, + "loss": 0.42396705627441406, + "mean_token_accuracy": 0.8686600789427757, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5668759573996067, + "epoch": 3.6741839762611277, + "grad_norm": 1.1999047994613647, + "learning_rate": 6.565927043103079e-05, + "loss": 0.42777458190917966, + "mean_token_accuracy": 0.8663509142398834, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5705659487843513, + "epoch": 3.792878338278932, + "grad_norm": 1.1414515972137451, + "learning_rate": 6.427253597036095e-05, + "loss": 0.42880672454833985, + "mean_token_accuracy": 0.8653362435102463, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.558414245545864, + "epoch": 3.9115727002967358, + "grad_norm": 1.3412097692489624, + "learning_rate": 6.284623584838158e-05, + "loss": 0.4282422256469727, + "mean_token_accuracy": 0.866187039911747, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4868207575633512, + "eval_loss": 0.6648371815681458, + "eval_mean_token_accuracy": 0.8163467548046893, + "eval_num_tokens": 2238308.0, + "eval_runtime": 111.72, + "eval_samples_per_second": 12.209, + "eval_steps_per_second": 1.531, + "step": 1688 + }, + { + "entropy": 0.546174580978258, + "epoch": 4.028486646884273, + "grad_norm": 1.464382290840149, + "learning_rate": 6.138280944164903e-05, + "loss": 0.40503074645996096, + "mean_token_accuracy": 0.8718915990161412, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.45237040892243385, + "epoch": 4.147181008902077, + "grad_norm": 1.7151323556900024, + "learning_rate": 5.988475962316552e-05, + "loss": 0.3065692901611328, + "mean_token_accuracy": 0.900569304227829, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.44834635987877847, + "epoch": 4.265875370919881, + "grad_norm": 1.2050637006759644, + "learning_rate": 5.835464848175874e-05, + "loss": 0.30684595108032225, + "mean_token_accuracy": 0.9003708437085152, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4549902780354023, + "epoch": 4.384569732937686, + "grad_norm": 1.20978844165802, + "learning_rate": 5.679509294018524e-05, + "loss": 0.3107210350036621, + "mean_token_accuracy": 0.8997164958715439, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.46374980479478833, + "epoch": 4.503264094955489, + "grad_norm": 1.0553879737854004, + "learning_rate": 5.520876027945252e-05, + "loss": 0.3163416862487793, + "mean_token_accuracy": 0.8980184662342071, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.4559279951453209, + "epoch": 4.621958456973294, + "grad_norm": 1.2723990678787231, + "learning_rate": 5.359836357701423e-05, + "loss": 0.31503250122070314, + "mean_token_accuracy": 0.8980488586425781, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.45531487330794335, + "epoch": 4.740652818991098, + "grad_norm": 1.3452478647232056, + "learning_rate": 5.1966657066640514e-05, + "loss": 0.3135023880004883, + "mean_token_accuracy": 0.8982085168361664, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.4609416849911213, + "epoch": 4.859347181008902, + "grad_norm": 1.37790846824646, + "learning_rate": 5.0316431427899296e-05, + "loss": 0.3144682502746582, + "mean_token_accuracy": 0.8983592641353607, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.45369990602135657, + "epoch": 4.978041543026706, + "grad_norm": 1.4144543409347534, + "learning_rate": 4.865050901330515e-05, + "loss": 0.31526716232299806, + "mean_token_accuracy": 0.8976324373483657, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4259050552956542, + "eval_loss": 0.7592839002609253, + "eval_mean_token_accuracy": 0.8117133866973788, + "eval_num_tokens": 2797885.0, + "eval_runtime": 111.9255, + "eval_samples_per_second": 12.187, + "eval_steps_per_second": 1.528, + "step": 2110 + }, + { + "entropy": 0.38868104307179524, + "epoch": 5.094955489614243, + "grad_norm": 1.2374166250228882, + "learning_rate": 4.697173902129824e-05, + "loss": 0.2337013626098633, + "mean_token_accuracy": 0.9221220642782105, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.3672105521708727, + "epoch": 5.213649851632048, + "grad_norm": 1.5339794158935547, + "learning_rate": 4.528299262330934e-05, + "loss": 0.21680095672607422, + "mean_token_accuracy": 0.9280403655767441, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3635194416344166, + "epoch": 5.332344213649852, + "grad_norm": 1.6350411176681519, + "learning_rate": 4.3587158053244715e-05, + "loss": 0.2198893928527832, + "mean_token_accuracy": 0.9272216448187828, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.36030806206166743, + "epoch": 5.451038575667655, + "grad_norm": 1.4245432615280151, + "learning_rate": 4.188713566778936e-05, + "loss": 0.21613813400268556, + "mean_token_accuracy": 0.9288999038934708, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.3618554932624102, + "epoch": 5.56973293768546, + "grad_norm": 1.609066367149353, + "learning_rate": 4.0185832985976856e-05, + "loss": 0.21525728225708007, + "mean_token_accuracy": 0.9276236236095429, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3609679951518774, + "epoch": 5.688427299703264, + "grad_norm": 1.0492284297943115, + "learning_rate": 3.848615971650962e-05, + "loss": 0.2146327590942383, + "mean_token_accuracy": 0.9283027359843254, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.34350181475281716, + "epoch": 5.807121661721069, + "grad_norm": 1.6522330045700073, + "learning_rate": 3.679102278133421e-05, + "loss": 0.20981479644775392, + "mean_token_accuracy": 0.9296323484182358, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.3616155734658241, + "epoch": 5.925816023738872, + "grad_norm": 1.762353777885437, + "learning_rate": 3.5103321343982645e-05, + "loss": 0.22292245864868165, + "mean_token_accuracy": 0.9254147034883499, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3478013425194032, + "eval_loss": 0.8838939666748047, + "eval_mean_token_accuracy": 0.8098702326155546, + "eval_num_tokens": 3357462.0, + "eval_runtime": 111.8466, + "eval_samples_per_second": 12.195, + "eval_steps_per_second": 1.529, + "step": 2532 + }, + { + "entropy": 0.33996166356929064, + "epoch": 6.04272997032641, + "grad_norm": 1.509031891822815, + "learning_rate": 3.342594185118299e-05, + "loss": 0.19556612014770508, + "mean_token_accuracy": 0.9348860970003351, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.29004726007580756, + "epoch": 6.161424332344214, + "grad_norm": 1.3547477722167969, + "learning_rate": 3.176175309621916e-05, + "loss": 0.1560215663909912, + "mean_token_accuracy": 0.9475028210878372, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.2834444552659988, + "epoch": 6.2801186943620175, + "grad_norm": 1.6714046001434326, + "learning_rate": 3.01136013124831e-05, + "loss": 0.15274467468261718, + "mean_token_accuracy": 0.9488504147529602, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.30035923302173617, + "epoch": 6.398813056379822, + "grad_norm": 1.149774432182312, + "learning_rate": 2.8484305305610797e-05, + "loss": 0.16209911346435546, + "mean_token_accuracy": 0.9469243207573891, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.28467367485165596, + "epoch": 6.517507418397626, + "grad_norm": 1.2987172603607178, + "learning_rate": 2.6876651632527578e-05, + "loss": 0.15439175605773925, + "mean_token_accuracy": 0.9486008253693581, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2905779552459717, + "epoch": 6.63620178041543, + "grad_norm": 1.3652619123458862, + "learning_rate": 2.529338983564765e-05, + "loss": 0.15736764907836914, + "mean_token_accuracy": 0.9474438297748565, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.27992172554135325, + "epoch": 6.7548961424332346, + "grad_norm": 1.092677354812622, + "learning_rate": 2.3737227740379025e-05, + "loss": 0.15728514671325683, + "mean_token_accuracy": 0.9473575839400291, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.28447474762797353, + "epoch": 6.873590504451038, + "grad_norm": 1.409013271331787, + "learning_rate": 2.221082682397646e-05, + "loss": 0.15529794692993165, + "mean_token_accuracy": 0.9479124927520752, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.2843698911368847, + "epoch": 6.992284866468843, + "grad_norm": 0.948100209236145, + "learning_rate": 2.0716797663662474e-05, + "loss": 0.15718685150146483, + "mean_token_accuracy": 0.9478301125764846, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.29428490229517396, + "eval_loss": 1.0146509408950806, + "eval_mean_token_accuracy": 0.8090401922750194, + "eval_num_tokens": 3917039.0, + "eval_runtime": 112.5894, + "eval_samples_per_second": 12.115, + "eval_steps_per_second": 1.519, + "step": 2954 + }, + { + "entropy": 0.26112703653761576, + "epoch": 7.10919881305638, + "grad_norm": 1.360489010810852, + "learning_rate": 1.9257695471802237e-05, + "loss": 0.1267384433746338, + "mean_token_accuracy": 0.957306801970235, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.2523047244548798, + "epoch": 7.227893175074184, + "grad_norm": 0.7781679630279541, + "learning_rate": 1.783601572576776e-05, + "loss": 0.12492096900939942, + "mean_token_accuracy": 0.9576244935393333, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.2563566393405199, + "epoch": 7.346587537091988, + "grad_norm": 1.2237826585769653, + "learning_rate": 1.645418989996586e-05, + "loss": 0.12833234786987305, + "mean_token_accuracy": 0.9578728675842285, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.24430972374975682, + "epoch": 7.465281899109792, + "grad_norm": 1.0145419836044312, + "learning_rate": 1.5114581307329361e-05, + "loss": 0.1226680850982666, + "mean_token_accuracy": 0.9584381604194641, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.2510662242770195, + "epoch": 7.583976261127597, + "grad_norm": 0.8724110722541809, + "learning_rate": 1.3819481057383636e-05, + "loss": 0.12699291229248047, + "mean_token_accuracy": 0.9558046141266823, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.2556200893968344, + "epoch": 7.7026706231454005, + "grad_norm": 1.4136648178100586, + "learning_rate": 1.2571104137801403e-05, + "loss": 0.13246843338012695, + "mean_token_accuracy": 0.9558334472775459, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.250102458037436, + "epoch": 7.821364985163205, + "grad_norm": 0.8711676597595215, + "learning_rate": 1.1371585626147424e-05, + "loss": 0.12776278495788573, + "mean_token_accuracy": 0.9560130417346955, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.2509234543889761, + "epoch": 7.940059347181009, + "grad_norm": 0.7930846214294434, + "learning_rate": 1.0222977038292094e-05, + "loss": 0.12907642364501953, + "mean_token_accuracy": 0.9563833594322204, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2663285747083307, + "eval_loss": 1.1223618984222412, + "eval_mean_token_accuracy": 0.8081985771307472, + "eval_num_tokens": 4476616.0, + "eval_runtime": 111.693, + "eval_samples_per_second": 12.212, + "eval_steps_per_second": 1.531, + "step": 3376 + }, + { + "entropy": 0.2438919508290775, + "epoch": 8.056973293768547, + "grad_norm": 0.5826248526573181, + "learning_rate": 9.12724281973914e-06, + "loss": 0.11967051506042481, + "mean_token_accuracy": 0.9597063037344646, + "num_tokens": 4510235.0, + "step": 3400 + }, + { + "entropy": 0.22686645958572627, + "epoch": 8.17566765578635, + "grad_norm": 0.5468573570251465, + "learning_rate": 8.086256985868273e-06, + "loss": 0.10963334083557129, + "mean_token_accuracy": 0.9632616007328033, + "num_tokens": 4577780.0, + "step": 3450 + }, + { + "entropy": 0.2430728640407324, + "epoch": 8.294362017804154, + "grad_norm": 0.6237483620643616, + "learning_rate": 7.101799916838946e-06, + "loss": 0.11639170646667481, + "mean_token_accuracy": 0.9603625151515007, + "num_tokens": 4641616.0, + "step": 3500 + }, + { + "entropy": 0.24073882170021535, + "epoch": 8.413056379821958, + "grad_norm": 0.6401548981666565, + "learning_rate": 6.175555312636793e-06, + "loss": 0.11829930305480957, + "mean_token_accuracy": 0.9581007561087609, + "num_tokens": 4706065.0, + "step": 3550 + }, + { + "entropy": 0.22724292811006308, + "epoch": 8.531750741839762, + "grad_norm": 0.45143744349479675, + "learning_rate": 5.309107313470484e-06, + "loss": 0.11297051429748535, + "mean_token_accuracy": 0.9609669655561447, + "num_tokens": 4772757.0, + "step": 3600 + }, + { + "entropy": 0.2373272515460849, + "epoch": 8.650445103857567, + "grad_norm": 0.4666413962841034, + "learning_rate": 4.5039377904440675e-06, + "loss": 0.11534086227416993, + "mean_token_accuracy": 0.9599428379535675, + "num_tokens": 4839113.0, + "step": 3650 + }, + { + "entropy": 0.23692427456378937, + "epoch": 8.769139465875371, + "grad_norm": 0.581846296787262, + "learning_rate": 3.7614238111382377e-06, + "loss": 0.11592741012573242, + "mean_token_accuracy": 0.9601775661110878, + "num_tokens": 4904798.0, + "step": 3700 + }, + { + "entropy": 0.2240232530236244, + "epoch": 8.887833827893175, + "grad_norm": 0.5457125902175903, + "learning_rate": 3.08283528443536e-06, + "loss": 0.1130020809173584, + "mean_token_accuracy": 0.9610978499054909, + "num_tokens": 4972291.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24879027266948545, + "eval_loss": 1.2201728820800781, + "eval_mean_token_accuracy": 0.8093260432544508, + "eval_num_tokens": 5036193.0, + "eval_runtime": 112.3651, + "eval_samples_per_second": 12.139, + "eval_steps_per_second": 1.522, + "step": 3798 + }, + { + "entropy": 0.23117562765397395, + "epoch": 9.004747774480713, + "grad_norm": 0.36207982897758484, + "learning_rate": 2.4693327886162175e-06, + "loss": 0.11734880447387695, + "mean_token_accuracy": 0.9605179792128239, + "num_tokens": 5038884.0, + "step": 3800 + }, + { + "entropy": 0.22370050594210625, + "epoch": 9.123442136498516, + "grad_norm": 0.45108872652053833, + "learning_rate": 1.921965586442878e-06, + "loss": 0.10498027801513672, + "mean_token_accuracy": 0.9636571845412254, + "num_tokens": 5106551.0, + "step": 3850 + }, + { + "entropy": 0.22246103022247554, + "epoch": 9.24213649851632, + "grad_norm": 0.6113122701644897, + "learning_rate": 1.4416698306226712e-06, + "loss": 0.1067995548248291, + "mean_token_accuracy": 0.963948774933815, + "num_tokens": 5173365.0, + "step": 3900 + }, + { + "entropy": 0.22863472890108824, + "epoch": 9.360830860534124, + "grad_norm": 0.5709601640701294, + "learning_rate": 1.0292669627223582e-06, + "loss": 0.10811909675598144, + "mean_token_accuracy": 0.9630391398072242, + "num_tokens": 5239920.0, + "step": 3950 + }, + { + "entropy": 0.22306809466332198, + "epoch": 9.47952522255193, + "grad_norm": 0.41473081707954407, + "learning_rate": 6.854623082706876e-07, + "loss": 0.10607051849365234, + "mean_token_accuracy": 0.962740405201912, + "num_tokens": 5306257.0, + "step": 4000 + }, + { + "entropy": 0.22921511493623256, + "epoch": 9.598219584569733, + "grad_norm": 0.46638238430023193, + "learning_rate": 4.108438704523254e-07, + "loss": 0.11014264106750488, + "mean_token_accuracy": 0.9613883411884308, + "num_tokens": 5370776.0, + "step": 4050 + }, + { + "entropy": 0.2404804431274533, + "epoch": 9.716913946587537, + "grad_norm": 0.579004168510437, + "learning_rate": 2.0588132445612997e-07, + "loss": 0.1125742244720459, + "mean_token_accuracy": 0.9599473521113395, + "num_tokens": 5435090.0, + "step": 4100 + }, + { + "entropy": 0.2219710022583604, + "epoch": 9.835608308605341, + "grad_norm": 0.5244644284248352, + "learning_rate": 7.09252141977214e-08, + "loss": 0.10504316329956055, + "mean_token_accuracy": 0.9633023545145989, + "num_tokens": 5503461.0, + "step": 4150 + }, + { + "entropy": 0.22302119687199592, + "epoch": 9.954302670623145, + "grad_norm": 2.2896687984466553, + "learning_rate": 6.206352790299599e-09, + "loss": 0.10735097885131836, + "mean_token_accuracy": 0.9625873160362244, + "num_tokens": 5569757.0, + "step": 4200 + }, + { + "epoch": 10.0, + "eval_entropy": 0.24441508539238868, + "eval_loss": 1.2669159173965454, + "eval_mean_token_accuracy": 0.8084659548530801, + "eval_num_tokens": 5595770.0, + "eval_runtime": 111.57, + "eval_samples_per_second": 12.226, + "eval_steps_per_second": 1.533, + "step": 4220 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.0850196489607578e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..99b7756efb6660bc02b28814566af66ae73537ce --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04758632698976937, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ae3b98420fc00083865f9e6c163df7a6ecec314e --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-844/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 844, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.4865781700611116, + "epoch": 0.11869436201780416, + "grad_norm": 3.600203037261963, + "learning_rate": 9.55350170221182e-06, + "loss": 2.2448199462890623, + "mean_token_accuracy": 0.571574572622776, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 1.1757256650924683, + "epoch": 0.23738872403560832, + "grad_norm": 1.843338966369629, + "learning_rate": 1.9301972826917757e-05, + "loss": 1.0319316864013672, + "mean_token_accuracy": 0.7366894924640656, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 1.0132716038823129, + "epoch": 0.3560830860534125, + "grad_norm": 2.0066328048706055, + "learning_rate": 2.9050443951623695e-05, + "loss": 0.8726716613769532, + "mean_token_accuracy": 0.7669131025671959, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9535812222957611, + "epoch": 0.47477744807121663, + "grad_norm": 1.8880268335342407, + "learning_rate": 3.8798915076329635e-05, + "loss": 0.8159033966064453, + "mean_token_accuracy": 0.778084682226181, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.9149203181266785, + "epoch": 0.5934718100890207, + "grad_norm": 1.5609782934188843, + "learning_rate": 4.8547386201035576e-05, + "loss": 0.7810882568359375, + "mean_token_accuracy": 0.7836975249648094, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.9015455979108811, + "epoch": 0.712166172106825, + "grad_norm": 1.6056852340698242, + "learning_rate": 5.829585732574152e-05, + "loss": 0.7620333099365234, + "mean_token_accuracy": 0.7867170104384422, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8751798084378243, + "epoch": 0.8308605341246291, + "grad_norm": 1.457392930984497, + "learning_rate": 6.804432845044745e-05, + "loss": 0.7453135681152344, + "mean_token_accuracy": 0.7934546408057213, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8582844731211662, + "epoch": 0.9495548961424333, + "grad_norm": 1.325522541999817, + "learning_rate": 7.779279957515339e-05, + "loss": 0.7352320861816406, + "mean_token_accuracy": 0.7950925189256668, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.699295549364815, + "eval_loss": 0.6675883531570435, + "eval_mean_token_accuracy": 0.8100697049620555, + "eval_num_tokens": 559577.0, + "eval_runtime": 113.6401, + "eval_samples_per_second": 12.003, + "eval_steps_per_second": 1.505, + "step": 422 + }, + { + "entropy": 0.8327096131852436, + "epoch": 1.0664688427299702, + "grad_norm": 0.9726872444152832, + "learning_rate": 8.226683697984623e-05, + "loss": 0.6989453887939453, + "mean_token_accuracy": 0.7999463737918641, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7919371470808982, + "epoch": 1.1851632047477745, + "grad_norm": 1.04917311668396, + "learning_rate": 8.219368143379697e-05, + "loss": 0.6691493225097657, + "mean_token_accuracy": 0.8088096314668656, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7981289568543434, + "epoch": 1.3038575667655787, + "grad_norm": 1.1144819259643555, + "learning_rate": 8.205030999972083e-05, + "loss": 0.6692163848876953, + "mean_token_accuracy": 0.8067614835500717, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.7944329422712326, + "epoch": 1.4225519287833828, + "grad_norm": 1.017581820487976, + "learning_rate": 8.183696788331456e-05, + "loss": 0.6635546112060546, + "mean_token_accuracy": 0.8121234861016273, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7757732102274895, + "epoch": 1.5412462908011868, + "grad_norm": 0.9428858160972595, + "learning_rate": 8.155401995992886e-05, + "loss": 0.6541387939453125, + "mean_token_accuracy": 0.8148220491409301, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7693249759078026, + "epoch": 1.659940652818991, + "grad_norm": 0.8914806842803955, + "learning_rate": 8.120195015052839e-05, + "loss": 0.6372745132446289, + "mean_token_accuracy": 0.8154754737019538, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7677849313616752, + "epoch": 1.7786350148367953, + "grad_norm": 1.2037655115127563, + "learning_rate": 8.078136059405015e-05, + "loss": 0.6468383026123047, + "mean_token_accuracy": 0.8125368970632553, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7699458369612694, + "epoch": 1.8973293768545996, + "grad_norm": 0.8997814059257507, + "learning_rate": 8.02929706175755e-05, + "loss": 0.6416602325439453, + "mean_token_accuracy": 0.8159067538380623, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6038547956455521, + "eval_loss": 0.6323259472846985, + "eval_mean_token_accuracy": 0.8163558896522076, + "eval_num_tokens": 1119154.0, + "eval_runtime": 111.6364, + "eval_samples_per_second": 12.218, + "eval_steps_per_second": 1.532, + "step": 844 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.158825213768499e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..770fc4bcccddee09f6766c99028592fc1c100c99 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: transformers +model_name: Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-4B-Base](https://huggingface.co/Qwen/Qwen3.5-4B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/8k3dxvta) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c1c6a2284809bfb3d576e2184806d304a1b536f3 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1266/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1266, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.389799338550067e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d91e1fd5db8b5c2bc857d21a220e01d09b02cb64 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1688/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1688, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.523636929350042e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8df5abf561e9e184202717ee9eda825981d53494 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2110/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2110, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0661414901345485e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0ea3c1c2541cbf16beed01bba6984dcd5e2c3e7f --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2532/trainer_state.json @@ -0,0 +1,600 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2532, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + }, + { + "entropy": 0.32862814035512466, + "epoch": 5.094955489614243, + "grad_norm": 0.8745105862617493, + "learning_rate": 0.00011038108617848764, + "loss": 0.2326393508911133, + "mean_token_accuracy": 0.9222641811153005, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.30950412914156916, + "epoch": 5.213649851632048, + "grad_norm": 0.9006625413894653, + "learning_rate": 0.00010641262204294646, + "loss": 0.21570093154907227, + "mean_token_accuracy": 0.9279602026939392, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3117005924880505, + "epoch": 5.332344213649852, + "grad_norm": 1.0628283023834229, + "learning_rate": 0.00010242750108036328, + "loss": 0.21999732971191407, + "mean_token_accuracy": 0.9277607050538063, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.3041132854670286, + "epoch": 5.451038575667655, + "grad_norm": 0.7804092168807983, + "learning_rate": 9.843253897454854e-05, + "loss": 0.21401237487792968, + "mean_token_accuracy": 0.9274822011590004, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.30939083263278006, + "epoch": 5.56973293768546, + "grad_norm": 1.147389531135559, + "learning_rate": 9.443456824045057e-05, + "loss": 0.21549640655517577, + "mean_token_accuracy": 0.9271116998791694, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3036996723711491, + "epoch": 5.688427299703264, + "grad_norm": 1.1045100688934326, + "learning_rate": 9.044042653862287e-05, + "loss": 0.21384597778320313, + "mean_token_accuracy": 0.9279281708598137, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.28685621835291386, + "epoch": 5.807121661721069, + "grad_norm": 0.8823838233947754, + "learning_rate": 8.645694498089104e-05, + "loss": 0.21053375244140626, + "mean_token_accuracy": 0.9295052057504654, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.30239632427692414, + "epoch": 5.925816023738872, + "grad_norm": 1.0655534267425537, + "learning_rate": 8.249093644721951e-05, + "loss": 0.22413877487182618, + "mean_token_accuracy": 0.9236766722798347, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3407397674538239, + "eval_loss": 0.8785887360572815, + "eval_mean_token_accuracy": 0.8086788755411293, + "eval_num_tokens": 3357462.0, + "eval_runtime": 109.1572, + "eval_samples_per_second": 12.487, + "eval_steps_per_second": 1.567, + "step": 2532 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.2784203268337664e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e4e1b351f709bf1ff00b10ce701986e2ecf7e00b --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-2954/trainer_state.json @@ -0,0 +1,701 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2954, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + }, + { + "entropy": 0.32862814035512466, + "epoch": 5.094955489614243, + "grad_norm": 0.8745105862617493, + "learning_rate": 0.00011038108617848764, + "loss": 0.2326393508911133, + "mean_token_accuracy": 0.9222641811153005, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.30950412914156916, + "epoch": 5.213649851632048, + "grad_norm": 0.9006625413894653, + "learning_rate": 0.00010641262204294646, + "loss": 0.21570093154907227, + "mean_token_accuracy": 0.9279602026939392, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3117005924880505, + "epoch": 5.332344213649852, + "grad_norm": 1.0628283023834229, + "learning_rate": 0.00010242750108036328, + "loss": 0.21999732971191407, + "mean_token_accuracy": 0.9277607050538063, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.3041132854670286, + "epoch": 5.451038575667655, + "grad_norm": 0.7804092168807983, + "learning_rate": 9.843253897454854e-05, + "loss": 0.21401237487792968, + "mean_token_accuracy": 0.9274822011590004, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.30939083263278006, + "epoch": 5.56973293768546, + "grad_norm": 1.147389531135559, + "learning_rate": 9.443456824045057e-05, + "loss": 0.21549640655517577, + "mean_token_accuracy": 0.9271116998791694, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3036996723711491, + "epoch": 5.688427299703264, + "grad_norm": 1.1045100688934326, + "learning_rate": 9.044042653862287e-05, + "loss": 0.21384597778320313, + "mean_token_accuracy": 0.9279281708598137, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.28685621835291386, + "epoch": 5.807121661721069, + "grad_norm": 0.8823838233947754, + "learning_rate": 8.645694498089104e-05, + "loss": 0.21053375244140626, + "mean_token_accuracy": 0.9295052057504654, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.30239632427692414, + "epoch": 5.925816023738872, + "grad_norm": 1.0655534267425537, + "learning_rate": 8.249093644721951e-05, + "loss": 0.22413877487182618, + "mean_token_accuracy": 0.9236766722798347, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3407397674538239, + "eval_loss": 0.8785887360572815, + "eval_mean_token_accuracy": 0.8086788755411293, + "eval_num_tokens": 3357462.0, + "eval_runtime": 109.1572, + "eval_samples_per_second": 12.487, + "eval_steps_per_second": 1.567, + "step": 2532 + }, + { + "entropy": 0.28097143368370037, + "epoch": 6.04272997032641, + "grad_norm": 0.9728150963783264, + "learning_rate": 7.85491839337604e-05, + "loss": 0.19440906524658202, + "mean_token_accuracy": 0.9360078654918574, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.22889940962195396, + "epoch": 6.161424332344214, + "grad_norm": 0.45930734276771545, + "learning_rate": 7.463842895201192e-05, + "loss": 0.15133169174194336, + "mean_token_accuracy": 0.9483904480934143, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.23173855409026145, + "epoch": 6.2801186943620175, + "grad_norm": 0.8016397953033447, + "learning_rate": 7.076535999892698e-05, + "loss": 0.14967309951782226, + "mean_token_accuracy": 0.9498896810412407, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.2416935010254383, + "epoch": 6.398813056379822, + "grad_norm": 0.9288640022277832, + "learning_rate": 6.693660111769221e-05, + "loss": 0.1576872444152832, + "mean_token_accuracy": 0.9475035279989242, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.23217199012637138, + "epoch": 6.517507418397626, + "grad_norm": 0.7940613031387329, + "learning_rate": 6.315870056874088e-05, + "loss": 0.15047475814819336, + "mean_token_accuracy": 0.950334153175354, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2362543848901987, + "epoch": 6.63620178041543, + "grad_norm": 0.5860735177993774, + "learning_rate": 5.943811963037561e-05, + "loss": 0.15330224990844726, + "mean_token_accuracy": 0.9486275720596313, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.2213127866387367, + "epoch": 6.7548961424332346, + "grad_norm": 0.7674234509468079, + "learning_rate": 5.578122154815522e-05, + "loss": 0.1498202419281006, + "mean_token_accuracy": 0.9496965748071671, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.22923343293368817, + "epoch": 6.873590504451038, + "grad_norm": 0.8681125044822693, + "learning_rate": 5.2194260651945735e-05, + "loss": 0.1494236946105957, + "mean_token_accuracy": 0.9495468598604202, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.22769563436508178, + "epoch": 6.992284866468843, + "grad_norm": 0.5667356252670288, + "learning_rate": 4.868337165924705e-05, + "loss": 0.14884186744689942, + "mean_token_accuracy": 0.9498968213796616, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.270330203032633, + "eval_loss": 1.0030295848846436, + "eval_mean_token_accuracy": 0.8112162616517808, + "eval_num_tokens": 3917039.0, + "eval_runtime": 108.9742, + "eval_samples_per_second": 12.508, + "eval_steps_per_second": 1.569, + "step": 2954 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.492274418430034e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e966e608f17c61a91be1f458d2a80fb924fd2576 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3376/trainer_state.json @@ -0,0 +1,792 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3376, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + }, + { + "entropy": 0.32862814035512466, + "epoch": 5.094955489614243, + "grad_norm": 0.8745105862617493, + "learning_rate": 0.00011038108617848764, + "loss": 0.2326393508911133, + "mean_token_accuracy": 0.9222641811153005, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.30950412914156916, + "epoch": 5.213649851632048, + "grad_norm": 0.9006625413894653, + "learning_rate": 0.00010641262204294646, + "loss": 0.21570093154907227, + "mean_token_accuracy": 0.9279602026939392, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3117005924880505, + "epoch": 5.332344213649852, + "grad_norm": 1.0628283023834229, + "learning_rate": 0.00010242750108036328, + "loss": 0.21999732971191407, + "mean_token_accuracy": 0.9277607050538063, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.3041132854670286, + "epoch": 5.451038575667655, + "grad_norm": 0.7804092168807983, + "learning_rate": 9.843253897454854e-05, + "loss": 0.21401237487792968, + "mean_token_accuracy": 0.9274822011590004, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.30939083263278006, + "epoch": 5.56973293768546, + "grad_norm": 1.147389531135559, + "learning_rate": 9.443456824045057e-05, + "loss": 0.21549640655517577, + "mean_token_accuracy": 0.9271116998791694, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3036996723711491, + "epoch": 5.688427299703264, + "grad_norm": 1.1045100688934326, + "learning_rate": 9.044042653862287e-05, + "loss": 0.21384597778320313, + "mean_token_accuracy": 0.9279281708598137, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.28685621835291386, + "epoch": 5.807121661721069, + "grad_norm": 0.8823838233947754, + "learning_rate": 8.645694498089104e-05, + "loss": 0.21053375244140626, + "mean_token_accuracy": 0.9295052057504654, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.30239632427692414, + "epoch": 5.925816023738872, + "grad_norm": 1.0655534267425537, + "learning_rate": 8.249093644721951e-05, + "loss": 0.22413877487182618, + "mean_token_accuracy": 0.9236766722798347, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3407397674538239, + "eval_loss": 0.8785887360572815, + "eval_mean_token_accuracy": 0.8086788755411293, + "eval_num_tokens": 3357462.0, + "eval_runtime": 109.1572, + "eval_samples_per_second": 12.487, + "eval_steps_per_second": 1.567, + "step": 2532 + }, + { + "entropy": 0.28097143368370037, + "epoch": 6.04272997032641, + "grad_norm": 0.9728150963783264, + "learning_rate": 7.85491839337604e-05, + "loss": 0.19440906524658202, + "mean_token_accuracy": 0.9360078654918574, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.22889940962195396, + "epoch": 6.161424332344214, + "grad_norm": 0.45930734276771545, + "learning_rate": 7.463842895201192e-05, + "loss": 0.15133169174194336, + "mean_token_accuracy": 0.9483904480934143, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.23173855409026145, + "epoch": 6.2801186943620175, + "grad_norm": 0.8016397953033447, + "learning_rate": 7.076535999892698e-05, + "loss": 0.14967309951782226, + "mean_token_accuracy": 0.9498896810412407, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.2416935010254383, + "epoch": 6.398813056379822, + "grad_norm": 0.9288640022277832, + "learning_rate": 6.693660111769221e-05, + "loss": 0.1576872444152832, + "mean_token_accuracy": 0.9475035279989242, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.23217199012637138, + "epoch": 6.517507418397626, + "grad_norm": 0.7940613031387329, + "learning_rate": 6.315870056874088e-05, + "loss": 0.15047475814819336, + "mean_token_accuracy": 0.950334153175354, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2362543848901987, + "epoch": 6.63620178041543, + "grad_norm": 0.5860735177993774, + "learning_rate": 5.943811963037561e-05, + "loss": 0.15330224990844726, + "mean_token_accuracy": 0.9486275720596313, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.2213127866387367, + "epoch": 6.7548961424332346, + "grad_norm": 0.7674234509468079, + "learning_rate": 5.578122154815522e-05, + "loss": 0.1498202419281006, + "mean_token_accuracy": 0.9496965748071671, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.22923343293368817, + "epoch": 6.873590504451038, + "grad_norm": 0.8681125044822693, + "learning_rate": 5.2194260651945735e-05, + "loss": 0.1494236946105957, + "mean_token_accuracy": 0.9495468598604202, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.22769563436508178, + "epoch": 6.992284866468843, + "grad_norm": 0.5667356252670288, + "learning_rate": 4.868337165924705e-05, + "loss": 0.14884186744689942, + "mean_token_accuracy": 0.9498968213796616, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.270330203032633, + "eval_loss": 1.0030295848846436, + "eval_mean_token_accuracy": 0.8112162616517808, + "eval_num_tokens": 3917039.0, + "eval_runtime": 108.9742, + "eval_samples_per_second": 12.508, + "eval_steps_per_second": 1.569, + "step": 2954 + }, + { + "entropy": 0.19700619601053634, + "epoch": 7.10919881305638, + "grad_norm": 0.3573840260505676, + "learning_rate": 4.52545591830916e-05, + "loss": 0.12250999450683593, + "mean_token_accuracy": 0.9572683457795739, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.19070270285010338, + "epoch": 7.227893175074184, + "grad_norm": 0.6056930422782898, + "learning_rate": 4.1913687462457894e-05, + "loss": 0.12080193519592285, + "mean_token_accuracy": 0.9578254294395446, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.1955683609098196, + "epoch": 7.346587537091988, + "grad_norm": 0.433248907327652, + "learning_rate": 3.866647033276339e-05, + "loss": 0.12155046463012695, + "mean_token_accuracy": 0.9586705946922303, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.18965637758374215, + "epoch": 7.465281899109792, + "grad_norm": 0.711413562297821, + "learning_rate": 3.551846145359021e-05, + "loss": 0.11862675666809082, + "mean_token_accuracy": 0.95881567299366, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.19531580347567798, + "epoch": 7.583976261127597, + "grad_norm": 0.3873218595981598, + "learning_rate": 3.247504481035671e-05, + "loss": 0.12266541481018066, + "mean_token_accuracy": 0.9565230271220208, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.19565934725105763, + "epoch": 7.7026706231454005, + "grad_norm": 0.39117226004600525, + "learning_rate": 2.954142550617977e-05, + "loss": 0.12585229873657228, + "mean_token_accuracy": 0.9569775006175041, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.19589349523186683, + "epoch": 7.821364985163205, + "grad_norm": 0.7185224294662476, + "learning_rate": 2.6722620859676613e-05, + "loss": 0.12319536209106445, + "mean_token_accuracy": 0.956814968585968, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.19540903151035308, + "epoch": 7.940059347181009, + "grad_norm": 0.39446574449539185, + "learning_rate": 2.4023451823931042e-05, + "loss": 0.12452852249145507, + "mean_token_accuracy": 0.9568252378702163, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24320883560947507, + "eval_loss": 1.0971084833145142, + "eval_mean_token_accuracy": 0.812525847501922, + "eval_num_tokens": 4476616.0, + "eval_runtime": 108.8684, + "eval_samples_per_second": 12.52, + "eval_steps_per_second": 1.571, + "step": 3376 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.704203198485463e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..169636864376b87eeae077fa50bc5dfec5eaf7ac --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-3798/trainer_state.json @@ -0,0 +1,883 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3798, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + }, + { + "entropy": 0.32862814035512466, + "epoch": 5.094955489614243, + "grad_norm": 0.8745105862617493, + "learning_rate": 0.00011038108617848764, + "loss": 0.2326393508911133, + "mean_token_accuracy": 0.9222641811153005, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.30950412914156916, + "epoch": 5.213649851632048, + "grad_norm": 0.9006625413894653, + "learning_rate": 0.00010641262204294646, + "loss": 0.21570093154907227, + "mean_token_accuracy": 0.9279602026939392, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3117005924880505, + "epoch": 5.332344213649852, + "grad_norm": 1.0628283023834229, + "learning_rate": 0.00010242750108036328, + "loss": 0.21999732971191407, + "mean_token_accuracy": 0.9277607050538063, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.3041132854670286, + "epoch": 5.451038575667655, + "grad_norm": 0.7804092168807983, + "learning_rate": 9.843253897454854e-05, + "loss": 0.21401237487792968, + "mean_token_accuracy": 0.9274822011590004, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.30939083263278006, + "epoch": 5.56973293768546, + "grad_norm": 1.147389531135559, + "learning_rate": 9.443456824045057e-05, + "loss": 0.21549640655517577, + "mean_token_accuracy": 0.9271116998791694, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3036996723711491, + "epoch": 5.688427299703264, + "grad_norm": 1.1045100688934326, + "learning_rate": 9.044042653862287e-05, + "loss": 0.21384597778320313, + "mean_token_accuracy": 0.9279281708598137, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.28685621835291386, + "epoch": 5.807121661721069, + "grad_norm": 0.8823838233947754, + "learning_rate": 8.645694498089104e-05, + "loss": 0.21053375244140626, + "mean_token_accuracy": 0.9295052057504654, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.30239632427692414, + "epoch": 5.925816023738872, + "grad_norm": 1.0655534267425537, + "learning_rate": 8.249093644721951e-05, + "loss": 0.22413877487182618, + "mean_token_accuracy": 0.9236766722798347, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3407397674538239, + "eval_loss": 0.8785887360572815, + "eval_mean_token_accuracy": 0.8086788755411293, + "eval_num_tokens": 3357462.0, + "eval_runtime": 109.1572, + "eval_samples_per_second": 12.487, + "eval_steps_per_second": 1.567, + "step": 2532 + }, + { + "entropy": 0.28097143368370037, + "epoch": 6.04272997032641, + "grad_norm": 0.9728150963783264, + "learning_rate": 7.85491839337604e-05, + "loss": 0.19440906524658202, + "mean_token_accuracy": 0.9360078654918574, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.22889940962195396, + "epoch": 6.161424332344214, + "grad_norm": 0.45930734276771545, + "learning_rate": 7.463842895201192e-05, + "loss": 0.15133169174194336, + "mean_token_accuracy": 0.9483904480934143, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.23173855409026145, + "epoch": 6.2801186943620175, + "grad_norm": 0.8016397953033447, + "learning_rate": 7.076535999892698e-05, + "loss": 0.14967309951782226, + "mean_token_accuracy": 0.9498896810412407, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.2416935010254383, + "epoch": 6.398813056379822, + "grad_norm": 0.9288640022277832, + "learning_rate": 6.693660111769221e-05, + "loss": 0.1576872444152832, + "mean_token_accuracy": 0.9475035279989242, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.23217199012637138, + "epoch": 6.517507418397626, + "grad_norm": 0.7940613031387329, + "learning_rate": 6.315870056874088e-05, + "loss": 0.15047475814819336, + "mean_token_accuracy": 0.950334153175354, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2362543848901987, + "epoch": 6.63620178041543, + "grad_norm": 0.5860735177993774, + "learning_rate": 5.943811963037561e-05, + "loss": 0.15330224990844726, + "mean_token_accuracy": 0.9486275720596313, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.2213127866387367, + "epoch": 6.7548961424332346, + "grad_norm": 0.7674234509468079, + "learning_rate": 5.578122154815522e-05, + "loss": 0.1498202419281006, + "mean_token_accuracy": 0.9496965748071671, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.22923343293368817, + "epoch": 6.873590504451038, + "grad_norm": 0.8681125044822693, + "learning_rate": 5.2194260651945735e-05, + "loss": 0.1494236946105957, + "mean_token_accuracy": 0.9495468598604202, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.22769563436508178, + "epoch": 6.992284866468843, + "grad_norm": 0.5667356252670288, + "learning_rate": 4.868337165924705e-05, + "loss": 0.14884186744689942, + "mean_token_accuracy": 0.9498968213796616, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.270330203032633, + "eval_loss": 1.0030295848846436, + "eval_mean_token_accuracy": 0.8112162616517808, + "eval_num_tokens": 3917039.0, + "eval_runtime": 108.9742, + "eval_samples_per_second": 12.508, + "eval_steps_per_second": 1.569, + "step": 2954 + }, + { + "entropy": 0.19700619601053634, + "epoch": 7.10919881305638, + "grad_norm": 0.3573840260505676, + "learning_rate": 4.52545591830916e-05, + "loss": 0.12250999450683593, + "mean_token_accuracy": 0.9572683457795739, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.19070270285010338, + "epoch": 7.227893175074184, + "grad_norm": 0.6056930422782898, + "learning_rate": 4.1913687462457894e-05, + "loss": 0.12080193519592285, + "mean_token_accuracy": 0.9578254294395446, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.1955683609098196, + "epoch": 7.346587537091988, + "grad_norm": 0.433248907327652, + "learning_rate": 3.866647033276339e-05, + "loss": 0.12155046463012695, + "mean_token_accuracy": 0.9586705946922303, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.18965637758374215, + "epoch": 7.465281899109792, + "grad_norm": 0.711413562297821, + "learning_rate": 3.551846145359021e-05, + "loss": 0.11862675666809082, + "mean_token_accuracy": 0.95881567299366, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.19531580347567798, + "epoch": 7.583976261127597, + "grad_norm": 0.3873218595981598, + "learning_rate": 3.247504481035671e-05, + "loss": 0.12266541481018066, + "mean_token_accuracy": 0.9565230271220208, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.19565934725105763, + "epoch": 7.7026706231454005, + "grad_norm": 0.39117226004600525, + "learning_rate": 2.954142550617977e-05, + "loss": 0.12585229873657228, + "mean_token_accuracy": 0.9569775006175041, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.19589349523186683, + "epoch": 7.821364985163205, + "grad_norm": 0.7185224294662476, + "learning_rate": 2.6722620859676613e-05, + "loss": 0.12319536209106445, + "mean_token_accuracy": 0.956814968585968, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.19540903151035308, + "epoch": 7.940059347181009, + "grad_norm": 0.39446574449539185, + "learning_rate": 2.4023451823931042e-05, + "loss": 0.12452852249145507, + "mean_token_accuracy": 0.9568252378702163, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24320883560947507, + "eval_loss": 1.0971084833145142, + "eval_mean_token_accuracy": 0.812525847501922, + "eval_num_tokens": 4476616.0, + "eval_runtime": 108.8684, + "eval_samples_per_second": 12.52, + "eval_steps_per_second": 1.571, + "step": 3376 + }, + { + "entropy": 0.18311175635926008, + "epoch": 8.056973293768547, + "grad_norm": 0.268220990896225, + "learning_rate": 2.144853474130035e-05, + "loss": 0.11313743591308593, + "mean_token_accuracy": 0.9608384258856023, + "num_tokens": 4510235.0, + "step": 3400 + }, + { + "entropy": 0.17055062495172024, + "epoch": 8.17566765578635, + "grad_norm": 0.23939041793346405, + "learning_rate": 1.900227344816441e-05, + "loss": 0.10537042617797851, + "mean_token_accuracy": 0.9629408642649651, + "num_tokens": 4577780.0, + "step": 3450 + }, + { + "entropy": 0.17994323082268238, + "epoch": 8.294362017804154, + "grad_norm": 0.4678446352481842, + "learning_rate": 1.6688851743120112e-05, + "loss": 0.11252813339233399, + "mean_token_accuracy": 0.9604243114590645, + "num_tokens": 4641616.0, + "step": 3500 + }, + { + "entropy": 0.17911005921661854, + "epoch": 8.413056379821958, + "grad_norm": 0.26558583974838257, + "learning_rate": 1.4512226231502611e-05, + "loss": 0.1144176959991455, + "mean_token_accuracy": 0.9580793642997741, + "num_tokens": 4706065.0, + "step": 3550 + }, + { + "entropy": 0.1720335179939866, + "epoch": 8.531750741839762, + "grad_norm": 0.2294122874736786, + "learning_rate": 1.2476119558471215e-05, + "loss": 0.10873400688171386, + "mean_token_accuracy": 0.9615193870663643, + "num_tokens": 4772757.0, + "step": 3600 + }, + { + "entropy": 0.17544500950723887, + "epoch": 8.650445103857567, + "grad_norm": 0.2873317003250122, + "learning_rate": 1.058401404223363e-05, + "loss": 0.11159305572509766, + "mean_token_accuracy": 0.9597178348898887, + "num_tokens": 4839113.0, + "step": 3650 + }, + { + "entropy": 0.17603869404643774, + "epoch": 8.769139465875371, + "grad_norm": 0.2209075689315796, + "learning_rate": 8.839145718296848e-06, + "loss": 0.11196708679199219, + "mean_token_accuracy": 0.960137557387352, + "num_tokens": 4904798.0, + "step": 3700 + }, + { + "entropy": 0.168403005823493, + "epoch": 8.887833827893175, + "grad_norm": 0.30203160643577576, + "learning_rate": 7.244498804931342e-06, + "loss": 0.10881103515625, + "mean_token_accuracy": 0.9608970630168915, + "num_tokens": 4972291.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2284291034553483, + "eval_loss": 1.1856080293655396, + "eval_mean_token_accuracy": 0.8127730955854494, + "eval_num_tokens": 5036193.0, + "eval_runtime": 109.0517, + "eval_samples_per_second": 12.499, + "eval_steps_per_second": 1.568, + "step": 3798 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9187313735163085e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4e0d6a2bd2c5ab00709dbe68f725546db4662b4f --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-422/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 422, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1351480592613376e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..69cf020a5d3feb494b041f3d6df5ed5c505aa38b --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4220/trainer_state.json @@ -0,0 +1,984 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + }, + { + "entropy": 0.7568841685498426, + "epoch": 2.0142433234421366, + "grad_norm": 1.1264771223068237, + "learning_rate": 0.00018737915163951255, + "loss": 0.6654921722412109, + "mean_token_accuracy": 0.8080597807308139, + "num_tokens": 1127140.0, + "step": 850 + }, + { + "entropy": 0.6357470096647739, + "epoch": 2.1329376854599404, + "grad_norm": 1.062138319015503, + "learning_rate": 0.0001859189642021757, + "loss": 0.5406727600097656, + "mean_token_accuracy": 0.8367875489592552, + "num_tokens": 1192314.0, + "step": 900 + }, + { + "entropy": 0.6489335931837559, + "epoch": 2.2516320474777447, + "grad_norm": 1.2298905849456787, + "learning_rate": 0.00018430614160886593, + "loss": 0.5534557342529297, + "mean_token_accuracy": 0.832711206972599, + "num_tokens": 1257658.0, + "step": 950 + }, + { + "entropy": 0.6525611525774002, + "epoch": 2.370326409495549, + "grad_norm": 1.240107536315918, + "learning_rate": 0.00018254344224231272, + "loss": 0.560151481628418, + "mean_token_accuracy": 0.8322853142023087, + "num_tokens": 1324523.0, + "step": 1000 + }, + { + "entropy": 0.6634468550980092, + "epoch": 2.489020771513353, + "grad_norm": 0.9729869961738586, + "learning_rate": 0.00018063388081691195, + "loss": 0.5720128631591797, + "mean_token_accuracy": 0.8308552953600884, + "num_tokens": 1391419.0, + "step": 1050 + }, + { + "entropy": 0.6703061027824879, + "epoch": 2.6077151335311575, + "grad_norm": 1.01613450050354, + "learning_rate": 0.0001785807232227117, + "loss": 0.572689208984375, + "mean_token_accuracy": 0.8295364084839821, + "num_tokens": 1456470.0, + "step": 1100 + }, + { + "entropy": 0.6784285299479962, + "epoch": 2.7264094955489613, + "grad_norm": 0.9121577143669128, + "learning_rate": 0.00017638748093981677, + "loss": 0.5718723678588867, + "mean_token_accuracy": 0.8284673491120338, + "num_tokens": 1523984.0, + "step": 1150 + }, + { + "entropy": 0.6573485015332698, + "epoch": 2.8451038575667655, + "grad_norm": 1.3088133335113525, + "learning_rate": 0.00017405790503276445, + "loss": 0.5624767684936524, + "mean_token_accuracy": 0.8323639875650406, + "num_tokens": 1591962.0, + "step": 1200 + }, + { + "entropy": 0.6734248518943786, + "epoch": 2.96379821958457, + "grad_norm": 0.9738526940345764, + "learning_rate": 0.0001715959797351436, + "loss": 0.5710682296752929, + "mean_token_accuracy": 0.82941819190979, + "num_tokens": 1658478.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5508473187859295, + "eval_loss": 0.691767156124115, + "eval_mean_token_accuracy": 0.8067876755842689, + "eval_num_tokens": 1678731.0, + "eval_runtime": 109.0685, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 1266 + }, + { + "entropy": 0.5739915644759455, + "epoch": 3.080712166172107, + "grad_norm": 1.2754778861999512, + "learning_rate": 0.00016900591563542844, + "loss": 0.4642959213256836, + "mean_token_accuracy": 0.8544093790998314, + "num_tokens": 1725154.0, + "step": 1300 + }, + { + "entropy": 0.5302996599674225, + "epoch": 3.199406528189911, + "grad_norm": 0.8599816560745239, + "learning_rate": 0.00016629214247568158, + "loss": 0.4227454376220703, + "mean_token_accuracy": 0.8642407047748566, + "num_tokens": 1792562.0, + "step": 1350 + }, + { + "entropy": 0.5344033451378345, + "epoch": 3.318100890207715, + "grad_norm": 1.090369701385498, + "learning_rate": 0.0001634593015754423, + "loss": 0.4355255889892578, + "mean_token_accuracy": 0.8621270033717156, + "num_tokens": 1858017.0, + "step": 1400 + }, + { + "entropy": 0.5277651423215866, + "epoch": 3.436795252225519, + "grad_norm": 1.4293097257614136, + "learning_rate": 0.00016051223789375756, + "loss": 0.43674335479736326, + "mean_token_accuracy": 0.8623704579472542, + "num_tokens": 1925739.0, + "step": 1450 + }, + { + "entropy": 0.5240404425561428, + "epoch": 3.5554896142433234, + "grad_norm": 1.1862913370132446, + "learning_rate": 0.0001574559917429322, + "loss": 0.4401676559448242, + "mean_token_accuracy": 0.8608517602086068, + "num_tokens": 1992896.0, + "step": 1500 + }, + { + "entropy": 0.5401698517799377, + "epoch": 3.6741839762611277, + "grad_norm": 1.0391490459442139, + "learning_rate": 0.00015429579016816914, + "loss": 0.45076507568359375, + "mean_token_accuracy": 0.8593063408136368, + "num_tokens": 2057970.0, + "step": 1550 + }, + { + "entropy": 0.5314346589148045, + "epoch": 3.792878338278932, + "grad_norm": 1.0559635162353516, + "learning_rate": 0.00015103703800784418, + "loss": 0.4557442092895508, + "mean_token_accuracy": 0.85676598072052, + "num_tokens": 2123534.0, + "step": 1600 + }, + { + "entropy": 0.5412077508866787, + "epoch": 3.9115727002967358, + "grad_norm": 3.4242255687713623, + "learning_rate": 0.00014768530864970382, + "loss": 0.45192554473876956, + "mean_token_accuracy": 0.8578489407896995, + "num_tokens": 2189979.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.48736624201835943, + "eval_loss": 0.7095903158187866, + "eval_mean_token_accuracy": 0.8107766930819952, + "eval_num_tokens": 2238308.0, + "eval_runtime": 109.0027, + "eval_samples_per_second": 12.504, + "eval_steps_per_second": 1.569, + "step": 1688 + }, + { + "entropy": 0.5045240473021105, + "epoch": 4.028486646884273, + "grad_norm": 1.3626340627670288, + "learning_rate": 0.00014424633449879628, + "loss": 0.42210906982421875, + "mean_token_accuracy": 0.8676861025355189, + "num_tokens": 2253247.0, + "step": 1700 + }, + { + "entropy": 0.39570884287357333, + "epoch": 4.147181008902077, + "grad_norm": 1.297686219215393, + "learning_rate": 0.00014072599717343732, + "loss": 0.30837583541870117, + "mean_token_accuracy": 0.8976307746767997, + "num_tokens": 2320148.0, + "step": 1750 + }, + { + "entropy": 0.4047955423593521, + "epoch": 4.265875370919881, + "grad_norm": 1.0024667978286743, + "learning_rate": 0.00013713031744598024, + "loss": 0.3086459350585937, + "mean_token_accuracy": 0.8986491727828979, + "num_tokens": 2386764.0, + "step": 1800 + }, + { + "entropy": 0.4188146521151066, + "epoch": 4.384569732937686, + "grad_norm": 1.1083835363388062, + "learning_rate": 0.00013346544494559215, + "loss": 0.31870101928710937, + "mean_token_accuracy": 0.8951004433631897, + "num_tokens": 2453770.0, + "step": 1850 + }, + { + "entropy": 0.42885379776358606, + "epoch": 4.503264094955489, + "grad_norm": 0.8239488005638123, + "learning_rate": 0.000129737647640649, + "loss": 0.31998380661010745, + "mean_token_accuracy": 0.8952840918302536, + "num_tokens": 2520198.0, + "step": 1900 + }, + { + "entropy": 0.42635195195674896, + "epoch": 4.621958456973294, + "grad_norm": 1.1509084701538086, + "learning_rate": 0.00012595330111873733, + "loss": 0.3244194030761719, + "mean_token_accuracy": 0.8931525626778603, + "num_tokens": 2587420.0, + "step": 1950 + }, + { + "entropy": 0.4147560016810894, + "epoch": 4.740652818991098, + "grad_norm": 1.1372978687286377, + "learning_rate": 0.0001221188776825964, + "loss": 0.32274589538574217, + "mean_token_accuracy": 0.8944549098610878, + "num_tokens": 2652728.0, + "step": 2000 + }, + { + "entropy": 0.42072165474295614, + "epoch": 4.859347181008902, + "grad_norm": 1.392776608467102, + "learning_rate": 0.00011824093528065022, + "loss": 0.3238159942626953, + "mean_token_accuracy": 0.894401975274086, + "num_tokens": 2718527.0, + "step": 2050 + }, + { + "entropy": 0.4175826041400433, + "epoch": 4.978041543026706, + "grad_norm": 0.947201669216156, + "learning_rate": 0.00011432610629106113, + "loss": 0.326630859375, + "mean_token_accuracy": 0.8935465425252914, + "num_tokens": 2785760.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.42097422114589755, + "eval_loss": 0.7806761264801025, + "eval_mean_token_accuracy": 0.8065890868504842, + "eval_num_tokens": 2797885.0, + "eval_runtime": 109.1077, + "eval_samples_per_second": 12.492, + "eval_steps_per_second": 1.567, + "step": 2110 + }, + { + "entropy": 0.32862814035512466, + "epoch": 5.094955489614243, + "grad_norm": 0.8745105862617493, + "learning_rate": 0.00011038108617848764, + "loss": 0.2326393508911133, + "mean_token_accuracy": 0.9222641811153005, + "num_tokens": 2850744.0, + "step": 2150 + }, + { + "entropy": 0.30950412914156916, + "epoch": 5.213649851632048, + "grad_norm": 0.9006625413894653, + "learning_rate": 0.00010641262204294646, + "loss": 0.21570093154907227, + "mean_token_accuracy": 0.9279602026939392, + "num_tokens": 2915946.0, + "step": 2200 + }, + { + "entropy": 0.3117005924880505, + "epoch": 5.332344213649852, + "grad_norm": 1.0628283023834229, + "learning_rate": 0.00010242750108036328, + "loss": 0.21999732971191407, + "mean_token_accuracy": 0.9277607050538063, + "num_tokens": 2981078.0, + "step": 2250 + }, + { + "entropy": 0.3041132854670286, + "epoch": 5.451038575667655, + "grad_norm": 0.7804092168807983, + "learning_rate": 9.843253897454854e-05, + "loss": 0.21401237487792968, + "mean_token_accuracy": 0.9274822011590004, + "num_tokens": 3048742.0, + "step": 2300 + }, + { + "entropy": 0.30939083263278006, + "epoch": 5.56973293768546, + "grad_norm": 1.147389531135559, + "learning_rate": 9.443456824045057e-05, + "loss": 0.21549640655517577, + "mean_token_accuracy": 0.9271116998791694, + "num_tokens": 3115199.0, + "step": 2350 + }, + { + "entropy": 0.3036996723711491, + "epoch": 5.688427299703264, + "grad_norm": 1.1045100688934326, + "learning_rate": 9.044042653862287e-05, + "loss": 0.21384597778320313, + "mean_token_accuracy": 0.9279281708598137, + "num_tokens": 3181331.0, + "step": 2400 + }, + { + "entropy": 0.28685621835291386, + "epoch": 5.807121661721069, + "grad_norm": 0.8823838233947754, + "learning_rate": 8.645694498089104e-05, + "loss": 0.21053375244140626, + "mean_token_accuracy": 0.9295052057504654, + "num_tokens": 3250226.0, + "step": 2450 + }, + { + "entropy": 0.30239632427692414, + "epoch": 5.925816023738872, + "grad_norm": 1.0655534267425537, + "learning_rate": 8.249093644721951e-05, + "loss": 0.22413877487182618, + "mean_token_accuracy": 0.9236766722798347, + "num_tokens": 3316528.0, + "step": 2500 + }, + { + "epoch": 6.0, + "eval_entropy": 0.3407397674538239, + "eval_loss": 0.8785887360572815, + "eval_mean_token_accuracy": 0.8086788755411293, + "eval_num_tokens": 3357462.0, + "eval_runtime": 109.1572, + "eval_samples_per_second": 12.487, + "eval_steps_per_second": 1.567, + "step": 2532 + }, + { + "entropy": 0.28097143368370037, + "epoch": 6.04272997032641, + "grad_norm": 0.9728150963783264, + "learning_rate": 7.85491839337604e-05, + "loss": 0.19440906524658202, + "mean_token_accuracy": 0.9360078654918574, + "num_tokens": 3381095.0, + "step": 2550 + }, + { + "entropy": 0.22889940962195396, + "epoch": 6.161424332344214, + "grad_norm": 0.45930734276771545, + "learning_rate": 7.463842895201192e-05, + "loss": 0.15133169174194336, + "mean_token_accuracy": 0.9483904480934143, + "num_tokens": 3446319.0, + "step": 2600 + }, + { + "entropy": 0.23173855409026145, + "epoch": 6.2801186943620175, + "grad_norm": 0.8016397953033447, + "learning_rate": 7.076535999892698e-05, + "loss": 0.14967309951782226, + "mean_token_accuracy": 0.9498896810412407, + "num_tokens": 3512109.0, + "step": 2650 + }, + { + "entropy": 0.2416935010254383, + "epoch": 6.398813056379822, + "grad_norm": 0.9288640022277832, + "learning_rate": 6.693660111769221e-05, + "loss": 0.1576872444152832, + "mean_token_accuracy": 0.9475035279989242, + "num_tokens": 3575841.0, + "step": 2700 + }, + { + "entropy": 0.23217199012637138, + "epoch": 6.517507418397626, + "grad_norm": 0.7940613031387329, + "learning_rate": 6.315870056874088e-05, + "loss": 0.15047475814819336, + "mean_token_accuracy": 0.950334153175354, + "num_tokens": 3643099.0, + "step": 2750 + }, + { + "entropy": 0.2362543848901987, + "epoch": 6.63620178041543, + "grad_norm": 0.5860735177993774, + "learning_rate": 5.943811963037561e-05, + "loss": 0.15330224990844726, + "mean_token_accuracy": 0.9486275720596313, + "num_tokens": 3709078.0, + "step": 2800 + }, + { + "entropy": 0.2213127866387367, + "epoch": 6.7548961424332346, + "grad_norm": 0.7674234509468079, + "learning_rate": 5.578122154815522e-05, + "loss": 0.1498202419281006, + "mean_token_accuracy": 0.9496965748071671, + "num_tokens": 3776816.0, + "step": 2850 + }, + { + "entropy": 0.22923343293368817, + "epoch": 6.873590504451038, + "grad_norm": 0.8681125044822693, + "learning_rate": 5.2194260651945735e-05, + "loss": 0.1494236946105957, + "mean_token_accuracy": 0.9495468598604202, + "num_tokens": 3845287.0, + "step": 2900 + }, + { + "entropy": 0.22769563436508178, + "epoch": 6.992284866468843, + "grad_norm": 0.5667356252670288, + "learning_rate": 4.868337165924705e-05, + "loss": 0.14884186744689942, + "mean_token_accuracy": 0.9498968213796616, + "num_tokens": 3912097.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.270330203032633, + "eval_loss": 1.0030295848846436, + "eval_mean_token_accuracy": 0.8112162616517808, + "eval_num_tokens": 3917039.0, + "eval_runtime": 108.9742, + "eval_samples_per_second": 12.508, + "eval_steps_per_second": 1.569, + "step": 2954 + }, + { + "entropy": 0.19700619601053634, + "epoch": 7.10919881305638, + "grad_norm": 0.3573840260505676, + "learning_rate": 4.52545591830916e-05, + "loss": 0.12250999450683593, + "mean_token_accuracy": 0.9572683457795739, + "num_tokens": 3976790.0, + "step": 3000 + }, + { + "entropy": 0.19070270285010338, + "epoch": 7.227893175074184, + "grad_norm": 0.6056930422782898, + "learning_rate": 4.1913687462457894e-05, + "loss": 0.12080193519592285, + "mean_token_accuracy": 0.9578254294395446, + "num_tokens": 4044284.0, + "step": 3050 + }, + { + "entropy": 0.1955683609098196, + "epoch": 7.346587537091988, + "grad_norm": 0.433248907327652, + "learning_rate": 3.866647033276339e-05, + "loss": 0.12155046463012695, + "mean_token_accuracy": 0.9586705946922303, + "num_tokens": 4110108.0, + "step": 3100 + }, + { + "entropy": 0.18965637758374215, + "epoch": 7.465281899109792, + "grad_norm": 0.711413562297821, + "learning_rate": 3.551846145359021e-05, + "loss": 0.11862675666809082, + "mean_token_accuracy": 0.95881567299366, + "num_tokens": 4178216.0, + "step": 3150 + }, + { + "entropy": 0.19531580347567798, + "epoch": 7.583976261127597, + "grad_norm": 0.3873218595981598, + "learning_rate": 3.247504481035671e-05, + "loss": 0.12266541481018066, + "mean_token_accuracy": 0.9565230271220208, + "num_tokens": 4244688.0, + "step": 3200 + }, + { + "entropy": 0.19565934725105763, + "epoch": 7.7026706231454005, + "grad_norm": 0.39117226004600525, + "learning_rate": 2.954142550617977e-05, + "loss": 0.12585229873657228, + "mean_token_accuracy": 0.9569775006175041, + "num_tokens": 4309445.0, + "step": 3250 + }, + { + "entropy": 0.19589349523186683, + "epoch": 7.821364985163205, + "grad_norm": 0.7185224294662476, + "learning_rate": 2.6722620859676613e-05, + "loss": 0.12319536209106445, + "mean_token_accuracy": 0.956814968585968, + "num_tokens": 4377115.0, + "step": 3300 + }, + { + "entropy": 0.19540903151035308, + "epoch": 7.940059347181009, + "grad_norm": 0.39446574449539185, + "learning_rate": 2.4023451823931042e-05, + "loss": 0.12452852249145507, + "mean_token_accuracy": 0.9568252378702163, + "num_tokens": 4443154.0, + "step": 3350 + }, + { + "epoch": 8.0, + "eval_entropy": 0.24320883560947507, + "eval_loss": 1.0971084833145142, + "eval_mean_token_accuracy": 0.812525847501922, + "eval_num_tokens": 4476616.0, + "eval_runtime": 108.8684, + "eval_samples_per_second": 12.52, + "eval_steps_per_second": 1.571, + "step": 3376 + }, + { + "entropy": 0.18311175635926008, + "epoch": 8.056973293768547, + "grad_norm": 0.268220990896225, + "learning_rate": 2.144853474130035e-05, + "loss": 0.11313743591308593, + "mean_token_accuracy": 0.9608384258856023, + "num_tokens": 4510235.0, + "step": 3400 + }, + { + "entropy": 0.17055062495172024, + "epoch": 8.17566765578635, + "grad_norm": 0.23939041793346405, + "learning_rate": 1.900227344816441e-05, + "loss": 0.10537042617797851, + "mean_token_accuracy": 0.9629408642649651, + "num_tokens": 4577780.0, + "step": 3450 + }, + { + "entropy": 0.17994323082268238, + "epoch": 8.294362017804154, + "grad_norm": 0.4678446352481842, + "learning_rate": 1.6688851743120112e-05, + "loss": 0.11252813339233399, + "mean_token_accuracy": 0.9604243114590645, + "num_tokens": 4641616.0, + "step": 3500 + }, + { + "entropy": 0.17911005921661854, + "epoch": 8.413056379821958, + "grad_norm": 0.26558583974838257, + "learning_rate": 1.4512226231502611e-05, + "loss": 0.1144176959991455, + "mean_token_accuracy": 0.9580793642997741, + "num_tokens": 4706065.0, + "step": 3550 + }, + { + "entropy": 0.1720335179939866, + "epoch": 8.531750741839762, + "grad_norm": 0.2294122874736786, + "learning_rate": 1.2476119558471215e-05, + "loss": 0.10873400688171386, + "mean_token_accuracy": 0.9615193870663643, + "num_tokens": 4772757.0, + "step": 3600 + }, + { + "entropy": 0.17544500950723887, + "epoch": 8.650445103857567, + "grad_norm": 0.2873317003250122, + "learning_rate": 1.058401404223363e-05, + "loss": 0.11159305572509766, + "mean_token_accuracy": 0.9597178348898887, + "num_tokens": 4839113.0, + "step": 3650 + }, + { + "entropy": 0.17603869404643774, + "epoch": 8.769139465875371, + "grad_norm": 0.2209075689315796, + "learning_rate": 8.839145718296848e-06, + "loss": 0.11196708679199219, + "mean_token_accuracy": 0.960137557387352, + "num_tokens": 4904798.0, + "step": 3700 + }, + { + "entropy": 0.168403005823493, + "epoch": 8.887833827893175, + "grad_norm": 0.30203160643577576, + "learning_rate": 7.244498804931342e-06, + "loss": 0.10881103515625, + "mean_token_accuracy": 0.9608970630168915, + "num_tokens": 4972291.0, + "step": 3750 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2284291034553483, + "eval_loss": 1.1856080293655396, + "eval_mean_token_accuracy": 0.8127730955854494, + "eval_num_tokens": 5036193.0, + "eval_runtime": 109.0517, + "eval_samples_per_second": 12.499, + "eval_steps_per_second": 1.568, + "step": 3798 + }, + { + "entropy": 0.17256613877521554, + "epoch": 9.004747774480713, + "grad_norm": 0.2025754600763321, + "learning_rate": 5.8028005993140365e-06, + "loss": 0.11303910255432129, + "mean_token_accuracy": 0.9607249946158549, + "num_tokens": 5038884.0, + "step": 3800 + }, + { + "entropy": 0.165403363853693, + "epoch": 9.123442136498516, + "grad_norm": 0.22349931299686432, + "learning_rate": 4.516516813078711e-06, + "loss": 0.10086984634399414, + "mean_token_accuracy": 0.9641102370619774, + "num_tokens": 5106551.0, + "step": 3850 + }, + { + "entropy": 0.1639752297475934, + "epoch": 9.24213649851632, + "grad_norm": 0.2920996844768524, + "learning_rate": 3.3878473552518804e-06, + "loss": 0.10313759803771973, + "mean_token_accuracy": 0.9639123389124871, + "num_tokens": 5173365.0, + "step": 3900 + }, + { + "entropy": 0.1668399443849921, + "epoch": 9.360830860534124, + "grad_norm": 0.2627892792224884, + "learning_rate": 2.4187225697863204e-06, + "loss": 0.10442820549011231, + "mean_token_accuracy": 0.9630197486281395, + "num_tokens": 5239920.0, + "step": 3950 + }, + { + "entropy": 0.16387769352644682, + "epoch": 9.47952522255193, + "grad_norm": 0.21084080636501312, + "learning_rate": 1.6107999341268723e-06, + "loss": 0.10237458229064941, + "mean_token_accuracy": 0.9630556285381318, + "num_tokens": 5306257.0, + "step": 4000 + }, + { + "entropy": 0.16678677145391702, + "epoch": 9.598219584569733, + "grad_norm": 0.26290667057037354, + "learning_rate": 9.654612244553885e-07, + "loss": 0.10608691215515137, + "mean_token_accuracy": 0.9614498975872994, + "num_tokens": 5370776.0, + "step": 4050 + }, + { + "entropy": 0.1743991317227483, + "epoch": 9.716913946587537, + "grad_norm": 0.33496949076652527, + "learning_rate": 4.838101524627172e-07, + "loss": 0.10892516136169433, + "mean_token_accuracy": 0.9598630359768867, + "num_tokens": 5435090.0, + "step": 4100 + }, + { + "entropy": 0.16130404870957135, + "epoch": 9.835608308605341, + "grad_norm": 0.26588767766952515, + "learning_rate": 1.6667047768950163e-07, + "loss": 0.10103833198547363, + "mean_token_accuracy": 0.9634099692106247, + "num_tokens": 5503461.0, + "step": 4150 + }, + { + "entropy": 0.16257579676806927, + "epoch": 9.954302670623145, + "grad_norm": 0.2929172217845917, + "learning_rate": 1.4584598664518915e-08, + "loss": 0.10325802803039551, + "mean_token_accuracy": 0.9627125361561775, + "num_tokens": 5569757.0, + "step": 4200 + }, + { + "epoch": 10.0, + "eval_entropy": 0.22572498738068586, + "eval_loss": 1.2293775081634521, + "eval_mean_token_accuracy": 0.8124386811116983, + "eval_num_tokens": 5595770.0, + "eval_runtime": 109.0662, + "eval_samples_per_second": 12.497, + "eval_steps_per_second": 1.568, + "step": 4220 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.133471377695703e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/README.md b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/adapter_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eddd8bd824923e3f3deb08780e53aae6e2c11ad4 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.014174918923300563, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "o_proj", + "gate_proj", + "v_proj", + "down_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/trainer_state.json b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bf2ed3a6bffd99bf8b1186167f399e250b2bb951 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-844/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 844, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8873072531819344, + "epoch": 0.11869436201780416, + "grad_norm": 2.190887451171875, + "learning_rate": 2.2450220423391633e-05, + "loss": 1.6781648254394532, + "mean_token_accuracy": 0.6478455939888954, + "num_tokens": 66201.0, + "step": 50 + }, + { + "entropy": 0.999365965127945, + "epoch": 0.23738872403560832, + "grad_norm": 1.6966168880462646, + "learning_rate": 4.535860861052595e-05, + "loss": 0.8697246551513672, + "mean_token_accuracy": 0.7676189291477203, + "num_tokens": 132943.0, + "step": 100 + }, + { + "entropy": 0.9288103449344635, + "epoch": 0.3560830860534125, + "grad_norm": 1.4732187986373901, + "learning_rate": 6.826699679766027e-05, + "loss": 0.8042455291748047, + "mean_token_accuracy": 0.7791835850477219, + "num_tokens": 202797.0, + "step": 150 + }, + { + "entropy": 0.9016453590989113, + "epoch": 0.47477744807121663, + "grad_norm": 1.38296377658844, + "learning_rate": 9.117538498479458e-05, + "loss": 0.779365234375, + "mean_token_accuracy": 0.7852362725138664, + "num_tokens": 266514.0, + "step": 200 + }, + { + "entropy": 0.8832121101021767, + "epoch": 0.5934718100890207, + "grad_norm": 1.0901048183441162, + "learning_rate": 0.00011408377317192892, + "loss": 0.7652861022949219, + "mean_token_accuracy": 0.7884920519590378, + "num_tokens": 332781.0, + "step": 250 + }, + { + "entropy": 0.8752336564660073, + "epoch": 0.712166172106825, + "grad_norm": 1.6169215440750122, + "learning_rate": 0.00013699216135906324, + "loss": 0.7603439331054688, + "mean_token_accuracy": 0.7882070636749268, + "num_tokens": 396012.0, + "step": 300 + }, + { + "entropy": 0.8663093891739845, + "epoch": 0.8308605341246291, + "grad_norm": 1.2983603477478027, + "learning_rate": 0.00015990054954619754, + "loss": 0.755434341430664, + "mean_token_accuracy": 0.7917704993486404, + "num_tokens": 464261.0, + "step": 350 + }, + { + "entropy": 0.8583173233270646, + "epoch": 0.9495548961424333, + "grad_norm": 1.1217254400253296, + "learning_rate": 0.00018280893773333186, + "loss": 0.7567882537841797, + "mean_token_accuracy": 0.7931881073117256, + "num_tokens": 531313.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7354744034901, + "eval_loss": 0.7186596989631653, + "eval_mean_token_accuracy": 0.7967988733659711, + "eval_num_tokens": 559577.0, + "eval_runtime": 110.6946, + "eval_samples_per_second": 12.313, + "eval_steps_per_second": 1.545, + "step": 422 + }, + { + "entropy": 0.8306438205810973, + "epoch": 1.0664688427299702, + "grad_norm": 1.2354347705841064, + "learning_rate": 0.00019332268746078505, + "loss": 0.7205262756347657, + "mean_token_accuracy": 0.7954642064680303, + "num_tokens": 595986.0, + "step": 450 + }, + { + "entropy": 0.7970015078783035, + "epoch": 1.1851632047477745, + "grad_norm": 1.088975191116333, + "learning_rate": 0.00019315077582197525, + "loss": 0.6885725402832031, + "mean_token_accuracy": 0.8067739251255989, + "num_tokens": 663734.0, + "step": 500 + }, + { + "entropy": 0.7895714005827904, + "epoch": 1.3038575667655787, + "grad_norm": 1.5963635444641113, + "learning_rate": 0.00019281386058421675, + "loss": 0.6921937561035156, + "mean_token_accuracy": 0.8038049748539925, + "num_tokens": 730600.0, + "step": 550 + }, + { + "entropy": 0.8042528739571572, + "epoch": 1.4225519287833828, + "grad_norm": 2.7907354831695557, + "learning_rate": 0.00019231251796784346, + "loss": 0.6985529327392578, + "mean_token_accuracy": 0.8045666691660881, + "num_tokens": 793502.0, + "step": 600 + }, + { + "entropy": 0.7797119709849357, + "epoch": 1.5412462908011868, + "grad_norm": 1.0708078145980835, + "learning_rate": 0.00019164760541050553, + "loss": 0.68753173828125, + "mean_token_accuracy": 0.8072108113765717, + "num_tokens": 860956.0, + "step": 650 + }, + { + "entropy": 0.7772400039434433, + "epoch": 1.659940652818991, + "grad_norm": 1.0909830331802368, + "learning_rate": 0.0001908202601007086, + "loss": 0.6747515869140625, + "mean_token_accuracy": 0.8066638374328613, + "num_tokens": 928211.0, + "step": 700 + }, + { + "entropy": 0.7770102015137672, + "epoch": 1.7786350148367953, + "grad_norm": 1.0903724431991577, + "learning_rate": 0.00018983189703289998, + "loss": 0.6874526214599609, + "mean_token_accuracy": 0.8048692178726197, + "num_tokens": 993921.0, + "step": 750 + }, + { + "entropy": 0.7821094547212124, + "epoch": 1.8973293768545996, + "grad_norm": 0.8551129102706909, + "learning_rate": 0.00018868420658742777, + "loss": 0.679476318359375, + "mean_token_accuracy": 0.8093244990706444, + "num_tokens": 1060568.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.61597507425219, + "eval_loss": 0.6859524846076965, + "eval_mean_token_accuracy": 0.8053807055043896, + "eval_num_tokens": 1119154.0, + "eval_runtime": 108.9909, + "eval_samples_per_second": 12.506, + "eval_steps_per_second": 1.569, + "step": 844 + } + ], + "logging_steps": 50, + "max_steps": 4220, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.255468078124237e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b78db3b6b1c4037f3aa928f161bc7b414cabec67 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1248/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1248, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.497980305197875e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ee3c662c4fb01b7462c69bdf9e5eb27d9c796429 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-1664/trainer_state.json @@ -0,0 +1,408 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1664, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.664604291013222e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8397100f729b9581ae60f859fcc2dbe3afbcb88e --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2080/trainer_state.json @@ -0,0 +1,499 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0850164994885837e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bf965645c630d5de82cc4037b13d7f3b62f5aa7c --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2496/trainer_state.json @@ -0,0 +1,590 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2496, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + }, + { + "entropy": 0.37054963635675836, + "epoch": 5.048134777376655, + "grad_norm": 0.9011105298995972, + "learning_rate": 0.00023261091391948947, + "loss": 0.28170028686523435, + "mean_token_accuracy": 0.9086048115383495, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.2936194147914648, + "epoch": 5.1684717208182915, + "grad_norm": 0.5512287616729736, + "learning_rate": 0.00022426303322199643, + "loss": 0.21786537170410156, + "mean_token_accuracy": 0.9288332989811897, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.2911713405698538, + "epoch": 5.2888086642599275, + "grad_norm": 0.5876255631446838, + "learning_rate": 0.00021587400939342754, + "loss": 0.2193107795715332, + "mean_token_accuracy": 0.9280375191569328, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.2986573086678982, + "epoch": 5.409145607701564, + "grad_norm": 0.46124088764190674, + "learning_rate": 0.00020745860683356717, + "loss": 0.22520872116088866, + "mean_token_accuracy": 0.9257059851288796, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.30760848000645635, + "epoch": 5.529482551143201, + "grad_norm": 0.44583287835121155, + "learning_rate": 0.00019903163636787952, + "loss": 0.22967565536499024, + "mean_token_accuracy": 0.9240611773729325, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.3116707868874073, + "epoch": 5.649819494584838, + "grad_norm": 0.4463329613208771, + "learning_rate": 0.00019060792918095394, + "loss": 0.22934612274169922, + "mean_token_accuracy": 0.9238958007097244, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.30317666091024875, + "epoch": 5.770156438026474, + "grad_norm": 0.5684349536895752, + "learning_rate": 0.00018220231071411927, + "loss": 0.22462085723876954, + "mean_token_accuracy": 0.9247557842731475, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.3070871848613024, + "epoch": 5.890493381468111, + "grad_norm": 0.4147462248802185, + "learning_rate": 0.00017382957457316484, + "loss": 0.22498483657836915, + "mean_token_accuracy": 0.9251977476477623, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.315427861636198, + "eval_loss": 0.772193193435669, + "eval_mean_token_accuracy": 0.8272767685636689, + "eval_num_tokens": 3517284.0, + "eval_runtime": 97.8925, + "eval_samples_per_second": 12.902, + "eval_steps_per_second": 1.614, + "step": 2496 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3025078142509875e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aa554e39fcf1d53155f77f080930f1cfac430f9c --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-2912/trainer_state.json @@ -0,0 +1,691 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2912, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + }, + { + "entropy": 0.37054963635675836, + "epoch": 5.048134777376655, + "grad_norm": 0.9011105298995972, + "learning_rate": 0.00023261091391948947, + "loss": 0.28170028686523435, + "mean_token_accuracy": 0.9086048115383495, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.2936194147914648, + "epoch": 5.1684717208182915, + "grad_norm": 0.5512287616729736, + "learning_rate": 0.00022426303322199643, + "loss": 0.21786537170410156, + "mean_token_accuracy": 0.9288332989811897, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.2911713405698538, + "epoch": 5.2888086642599275, + "grad_norm": 0.5876255631446838, + "learning_rate": 0.00021587400939342754, + "loss": 0.2193107795715332, + "mean_token_accuracy": 0.9280375191569328, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.2986573086678982, + "epoch": 5.409145607701564, + "grad_norm": 0.46124088764190674, + "learning_rate": 0.00020745860683356717, + "loss": 0.22520872116088866, + "mean_token_accuracy": 0.9257059851288796, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.30760848000645635, + "epoch": 5.529482551143201, + "grad_norm": 0.44583287835121155, + "learning_rate": 0.00019903163636787952, + "loss": 0.22967565536499024, + "mean_token_accuracy": 0.9240611773729325, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.3116707868874073, + "epoch": 5.649819494584838, + "grad_norm": 0.4463329613208771, + "learning_rate": 0.00019060792918095394, + "loss": 0.22934612274169922, + "mean_token_accuracy": 0.9238958007097244, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.30317666091024875, + "epoch": 5.770156438026474, + "grad_norm": 0.5684349536895752, + "learning_rate": 0.00018220231071411927, + "loss": 0.22462085723876954, + "mean_token_accuracy": 0.9247557842731475, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.3070871848613024, + "epoch": 5.890493381468111, + "grad_norm": 0.4147462248802185, + "learning_rate": 0.00017382957457316484, + "loss": 0.22498483657836915, + "mean_token_accuracy": 0.9251977476477623, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.315427861636198, + "eval_loss": 0.772193193435669, + "eval_mean_token_accuracy": 0.8272767685636689, + "eval_num_tokens": 3517284.0, + "eval_runtime": 97.8925, + "eval_samples_per_second": 12.902, + "eval_steps_per_second": 1.614, + "step": 2496 + }, + { + "entropy": 0.3059864859237815, + "epoch": 6.009626955475331, + "grad_norm": 0.2763276696205139, + "learning_rate": 0.00016550445649209263, + "loss": 0.21960899353027344, + "mean_token_accuracy": 0.9276399949584344, + "num_tokens": 3523144.0, + "step": 2500 + }, + { + "entropy": 0.22258748069405557, + "epoch": 6.129963898916968, + "grad_norm": 0.42732444405555725, + "learning_rate": 0.00015724160839872102, + "loss": 0.14948601722717286, + "mean_token_accuracy": 0.9504047948122024, + "num_tokens": 3594122.0, + "step": 2550 + }, + { + "entropy": 0.24060228884220122, + "epoch": 6.250300842358604, + "grad_norm": 0.473332941532135, + "learning_rate": 0.000149055572627784, + "loss": 0.15332001686096192, + "mean_token_accuracy": 0.9503934115171433, + "num_tokens": 3663655.0, + "step": 2600 + }, + { + "entropy": 0.2281983983516693, + "epoch": 6.37063778580024, + "grad_norm": 0.4745512306690216, + "learning_rate": 0.00014096075632691189, + "loss": 0.15419836044311525, + "mean_token_accuracy": 0.9501351490616798, + "num_tokens": 3733302.0, + "step": 2650 + }, + { + "entropy": 0.2444649949669838, + "epoch": 6.490974729241877, + "grad_norm": 0.38537177443504333, + "learning_rate": 0.0001329714061005355, + "loss": 0.15538623809814453, + "mean_token_accuracy": 0.9505197313427926, + "num_tokens": 3803551.0, + "step": 2700 + }, + { + "entropy": 0.2471274556219578, + "epoch": 6.611311672683514, + "grad_norm": 0.4447166323661804, + "learning_rate": 0.000125101582936342, + "loss": 0.15631080627441407, + "mean_token_accuracy": 0.9499771365523338, + "num_tokens": 3873200.0, + "step": 2750 + }, + { + "entropy": 0.23467195242643357, + "epoch": 6.731648616125151, + "grad_norm": 0.5019384622573853, + "learning_rate": 0.00011736513745840905, + "loss": 0.1504099941253662, + "mean_token_accuracy": 0.9516549646854401, + "num_tokens": 3943741.0, + "step": 2800 + }, + { + "entropy": 0.2275939880311489, + "epoch": 6.851985559566787, + "grad_norm": 0.45045986771583557, + "learning_rate": 0.00010977568555057236, + "loss": 0.15092979431152342, + "mean_token_accuracy": 0.9501178854703903, + "num_tokens": 4016857.0, + "step": 2850 + }, + { + "entropy": 0.23294402249157428, + "epoch": 6.9723225030084235, + "grad_norm": 0.3174738585948944, + "learning_rate": 0.00010234658439292866, + "loss": 0.15235508918762208, + "mean_token_accuracy": 0.9500771847367286, + "num_tokens": 4087868.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.272654819620561, + "eval_loss": 0.8500064015388489, + "eval_mean_token_accuracy": 0.8321780105934867, + "eval_num_tokens": 4103498.0, + "eval_runtime": 98.0906, + "eval_samples_per_second": 12.876, + "eval_steps_per_second": 1.611, + "step": 2912 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5183410722563686e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..17031ef30ecf18c0283742b9a1bae32b5a7c2fc4 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3328/trainer_state.json @@ -0,0 +1,782 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3328, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + }, + { + "entropy": 0.37054963635675836, + "epoch": 5.048134777376655, + "grad_norm": 0.9011105298995972, + "learning_rate": 0.00023261091391948947, + "loss": 0.28170028686523435, + "mean_token_accuracy": 0.9086048115383495, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.2936194147914648, + "epoch": 5.1684717208182915, + "grad_norm": 0.5512287616729736, + "learning_rate": 0.00022426303322199643, + "loss": 0.21786537170410156, + "mean_token_accuracy": 0.9288332989811897, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.2911713405698538, + "epoch": 5.2888086642599275, + "grad_norm": 0.5876255631446838, + "learning_rate": 0.00021587400939342754, + "loss": 0.2193107795715332, + "mean_token_accuracy": 0.9280375191569328, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.2986573086678982, + "epoch": 5.409145607701564, + "grad_norm": 0.46124088764190674, + "learning_rate": 0.00020745860683356717, + "loss": 0.22520872116088866, + "mean_token_accuracy": 0.9257059851288796, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.30760848000645635, + "epoch": 5.529482551143201, + "grad_norm": 0.44583287835121155, + "learning_rate": 0.00019903163636787952, + "loss": 0.22967565536499024, + "mean_token_accuracy": 0.9240611773729325, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.3116707868874073, + "epoch": 5.649819494584838, + "grad_norm": 0.4463329613208771, + "learning_rate": 0.00019060792918095394, + "loss": 0.22934612274169922, + "mean_token_accuracy": 0.9238958007097244, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.30317666091024875, + "epoch": 5.770156438026474, + "grad_norm": 0.5684349536895752, + "learning_rate": 0.00018220231071411927, + "loss": 0.22462085723876954, + "mean_token_accuracy": 0.9247557842731475, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.3070871848613024, + "epoch": 5.890493381468111, + "grad_norm": 0.4147462248802185, + "learning_rate": 0.00017382957457316484, + "loss": 0.22498483657836915, + "mean_token_accuracy": 0.9251977476477623, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.315427861636198, + "eval_loss": 0.772193193435669, + "eval_mean_token_accuracy": 0.8272767685636689, + "eval_num_tokens": 3517284.0, + "eval_runtime": 97.8925, + "eval_samples_per_second": 12.902, + "eval_steps_per_second": 1.614, + "step": 2496 + }, + { + "entropy": 0.3059864859237815, + "epoch": 6.009626955475331, + "grad_norm": 0.2763276696205139, + "learning_rate": 0.00016550445649209263, + "loss": 0.21960899353027344, + "mean_token_accuracy": 0.9276399949584344, + "num_tokens": 3523144.0, + "step": 2500 + }, + { + "entropy": 0.22258748069405557, + "epoch": 6.129963898916968, + "grad_norm": 0.42732444405555725, + "learning_rate": 0.00015724160839872102, + "loss": 0.14948601722717286, + "mean_token_accuracy": 0.9504047948122024, + "num_tokens": 3594122.0, + "step": 2550 + }, + { + "entropy": 0.24060228884220122, + "epoch": 6.250300842358604, + "grad_norm": 0.473332941532135, + "learning_rate": 0.000149055572627784, + "loss": 0.15332001686096192, + "mean_token_accuracy": 0.9503934115171433, + "num_tokens": 3663655.0, + "step": 2600 + }, + { + "entropy": 0.2281983983516693, + "epoch": 6.37063778580024, + "grad_norm": 0.4745512306690216, + "learning_rate": 0.00014096075632691189, + "loss": 0.15419836044311525, + "mean_token_accuracy": 0.9501351490616798, + "num_tokens": 3733302.0, + "step": 2650 + }, + { + "entropy": 0.2444649949669838, + "epoch": 6.490974729241877, + "grad_norm": 0.38537177443504333, + "learning_rate": 0.0001329714061005355, + "loss": 0.15538623809814453, + "mean_token_accuracy": 0.9505197313427926, + "num_tokens": 3803551.0, + "step": 2700 + }, + { + "entropy": 0.2471274556219578, + "epoch": 6.611311672683514, + "grad_norm": 0.4447166323661804, + "learning_rate": 0.000125101582936342, + "loss": 0.15631080627441407, + "mean_token_accuracy": 0.9499771365523338, + "num_tokens": 3873200.0, + "step": 2750 + }, + { + "entropy": 0.23467195242643357, + "epoch": 6.731648616125151, + "grad_norm": 0.5019384622573853, + "learning_rate": 0.00011736513745840905, + "loss": 0.1504099941253662, + "mean_token_accuracy": 0.9516549646854401, + "num_tokens": 3943741.0, + "step": 2800 + }, + { + "entropy": 0.2275939880311489, + "epoch": 6.851985559566787, + "grad_norm": 0.45045986771583557, + "learning_rate": 0.00010977568555057236, + "loss": 0.15092979431152342, + "mean_token_accuracy": 0.9501178854703903, + "num_tokens": 4016857.0, + "step": 2850 + }, + { + "entropy": 0.23294402249157428, + "epoch": 6.9723225030084235, + "grad_norm": 0.3174738585948944, + "learning_rate": 0.00010234658439292866, + "loss": 0.15235508918762208, + "mean_token_accuracy": 0.9500771847367286, + "num_tokens": 4087868.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.272654819620561, + "eval_loss": 0.8500064015388489, + "eval_mean_token_accuracy": 0.8321780105934867, + "eval_num_tokens": 4103498.0, + "eval_runtime": 98.0906, + "eval_samples_per_second": 12.876, + "eval_steps_per_second": 1.611, + "step": 2912 + }, + { + "entropy": 0.21614460315969256, + "epoch": 7.091456077015644, + "grad_norm": 0.23043400049209595, + "learning_rate": 9.50909089536471e-05, + "loss": 0.12937673568725586, + "mean_token_accuracy": 0.9584822886519961, + "num_tokens": 4154257.0, + "step": 2950 + }, + { + "entropy": 0.19473189041018485, + "epoch": 7.21179302045728, + "grad_norm": 0.27521875500679016, + "learning_rate": 8.802142897746567e-05, + "loss": 0.11380335807800293, + "mean_token_accuracy": 0.9625545349717141, + "num_tokens": 4226995.0, + "step": 3000 + }, + { + "entropy": 0.18990286618471144, + "epoch": 7.332129963898917, + "grad_norm": 0.2729308307170868, + "learning_rate": 8.115058651136956e-05, + "loss": 0.11597777366638183, + "mean_token_accuracy": 0.9613875389099121, + "num_tokens": 4298066.0, + "step": 3050 + }, + { + "entropy": 0.18902748223394156, + "epoch": 7.452466907340553, + "grad_norm": 0.3057991564273834, + "learning_rate": 7.449047400700742e-05, + "loss": 0.11589047431945801, + "mean_token_accuracy": 0.9611530381441117, + "num_tokens": 4370032.0, + "step": 3100 + }, + { + "entropy": 0.18580251831561326, + "epoch": 7.57280385078219, + "grad_norm": 0.3325570523738861, + "learning_rate": 6.805281303838264e-05, + "loss": 0.11706151008605957, + "mean_token_accuracy": 0.9613592675328255, + "num_tokens": 4440775.0, + "step": 3150 + }, + { + "entropy": 0.1917814151570201, + "epoch": 7.693140794223827, + "grad_norm": 0.2866221070289612, + "learning_rate": 6.184893367227762e-05, + "loss": 0.11704328536987305, + "mean_token_accuracy": 0.9623443368077278, + "num_tokens": 4512445.0, + "step": 3200 + }, + { + "entropy": 0.1835070162639022, + "epoch": 7.813477737665464, + "grad_norm": 0.4219290018081665, + "learning_rate": 5.5889754527717426e-05, + "loss": 0.11752543449401856, + "mean_token_accuracy": 0.9615215319395065, + "num_tokens": 4582939.0, + "step": 3250 + }, + { + "entropy": 0.19630121663212777, + "epoch": 7.9338146811070995, + "grad_norm": 0.20138691365718842, + "learning_rate": 5.018576355956703e-05, + "loss": 0.11969227790832519, + "mean_token_accuracy": 0.9607423830032349, + "num_tokens": 4651557.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.22974008284037625, + "eval_loss": 1.002966284751892, + "eval_mean_token_accuracy": 0.8310878254190276, + "eval_num_tokens": 4689712.0, + "eval_runtime": 97.8527, + "eval_samples_per_second": 12.907, + "eval_steps_per_second": 1.615, + "step": 3328 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.734165130055086e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3066a6526a8fe8f4500c4fc6d99119acb58e5187 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-3744/trainer_state.json @@ -0,0 +1,873 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3744, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + }, + { + "entropy": 0.37054963635675836, + "epoch": 5.048134777376655, + "grad_norm": 0.9011105298995972, + "learning_rate": 0.00023261091391948947, + "loss": 0.28170028686523435, + "mean_token_accuracy": 0.9086048115383495, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.2936194147914648, + "epoch": 5.1684717208182915, + "grad_norm": 0.5512287616729736, + "learning_rate": 0.00022426303322199643, + "loss": 0.21786537170410156, + "mean_token_accuracy": 0.9288332989811897, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.2911713405698538, + "epoch": 5.2888086642599275, + "grad_norm": 0.5876255631446838, + "learning_rate": 0.00021587400939342754, + "loss": 0.2193107795715332, + "mean_token_accuracy": 0.9280375191569328, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.2986573086678982, + "epoch": 5.409145607701564, + "grad_norm": 0.46124088764190674, + "learning_rate": 0.00020745860683356717, + "loss": 0.22520872116088866, + "mean_token_accuracy": 0.9257059851288796, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.30760848000645635, + "epoch": 5.529482551143201, + "grad_norm": 0.44583287835121155, + "learning_rate": 0.00019903163636787952, + "loss": 0.22967565536499024, + "mean_token_accuracy": 0.9240611773729325, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.3116707868874073, + "epoch": 5.649819494584838, + "grad_norm": 0.4463329613208771, + "learning_rate": 0.00019060792918095394, + "loss": 0.22934612274169922, + "mean_token_accuracy": 0.9238958007097244, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.30317666091024875, + "epoch": 5.770156438026474, + "grad_norm": 0.5684349536895752, + "learning_rate": 0.00018220231071411927, + "loss": 0.22462085723876954, + "mean_token_accuracy": 0.9247557842731475, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.3070871848613024, + "epoch": 5.890493381468111, + "grad_norm": 0.4147462248802185, + "learning_rate": 0.00017382957457316484, + "loss": 0.22498483657836915, + "mean_token_accuracy": 0.9251977476477623, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.315427861636198, + "eval_loss": 0.772193193435669, + "eval_mean_token_accuracy": 0.8272767685636689, + "eval_num_tokens": 3517284.0, + "eval_runtime": 97.8925, + "eval_samples_per_second": 12.902, + "eval_steps_per_second": 1.614, + "step": 2496 + }, + { + "entropy": 0.3059864859237815, + "epoch": 6.009626955475331, + "grad_norm": 0.2763276696205139, + "learning_rate": 0.00016550445649209263, + "loss": 0.21960899353027344, + "mean_token_accuracy": 0.9276399949584344, + "num_tokens": 3523144.0, + "step": 2500 + }, + { + "entropy": 0.22258748069405557, + "epoch": 6.129963898916968, + "grad_norm": 0.42732444405555725, + "learning_rate": 0.00015724160839872102, + "loss": 0.14948601722717286, + "mean_token_accuracy": 0.9504047948122024, + "num_tokens": 3594122.0, + "step": 2550 + }, + { + "entropy": 0.24060228884220122, + "epoch": 6.250300842358604, + "grad_norm": 0.473332941532135, + "learning_rate": 0.000149055572627784, + "loss": 0.15332001686096192, + "mean_token_accuracy": 0.9503934115171433, + "num_tokens": 3663655.0, + "step": 2600 + }, + { + "entropy": 0.2281983983516693, + "epoch": 6.37063778580024, + "grad_norm": 0.4745512306690216, + "learning_rate": 0.00014096075632691189, + "loss": 0.15419836044311525, + "mean_token_accuracy": 0.9501351490616798, + "num_tokens": 3733302.0, + "step": 2650 + }, + { + "entropy": 0.2444649949669838, + "epoch": 6.490974729241877, + "grad_norm": 0.38537177443504333, + "learning_rate": 0.0001329714061005355, + "loss": 0.15538623809814453, + "mean_token_accuracy": 0.9505197313427926, + "num_tokens": 3803551.0, + "step": 2700 + }, + { + "entropy": 0.2471274556219578, + "epoch": 6.611311672683514, + "grad_norm": 0.4447166323661804, + "learning_rate": 0.000125101582936342, + "loss": 0.15631080627441407, + "mean_token_accuracy": 0.9499771365523338, + "num_tokens": 3873200.0, + "step": 2750 + }, + { + "entropy": 0.23467195242643357, + "epoch": 6.731648616125151, + "grad_norm": 0.5019384622573853, + "learning_rate": 0.00011736513745840905, + "loss": 0.1504099941253662, + "mean_token_accuracy": 0.9516549646854401, + "num_tokens": 3943741.0, + "step": 2800 + }, + { + "entropy": 0.2275939880311489, + "epoch": 6.851985559566787, + "grad_norm": 0.45045986771583557, + "learning_rate": 0.00010977568555057236, + "loss": 0.15092979431152342, + "mean_token_accuracy": 0.9501178854703903, + "num_tokens": 4016857.0, + "step": 2850 + }, + { + "entropy": 0.23294402249157428, + "epoch": 6.9723225030084235, + "grad_norm": 0.3174738585948944, + "learning_rate": 0.00010234658439292866, + "loss": 0.15235508918762208, + "mean_token_accuracy": 0.9500771847367286, + "num_tokens": 4087868.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.272654819620561, + "eval_loss": 0.8500064015388489, + "eval_mean_token_accuracy": 0.8321780105934867, + "eval_num_tokens": 4103498.0, + "eval_runtime": 98.0906, + "eval_samples_per_second": 12.876, + "eval_steps_per_second": 1.611, + "step": 2912 + }, + { + "entropy": 0.21614460315969256, + "epoch": 7.091456077015644, + "grad_norm": 0.23043400049209595, + "learning_rate": 9.50909089536471e-05, + "loss": 0.12937673568725586, + "mean_token_accuracy": 0.9584822886519961, + "num_tokens": 4154257.0, + "step": 2950 + }, + { + "entropy": 0.19473189041018485, + "epoch": 7.21179302045728, + "grad_norm": 0.27521875500679016, + "learning_rate": 8.802142897746567e-05, + "loss": 0.11380335807800293, + "mean_token_accuracy": 0.9625545349717141, + "num_tokens": 4226995.0, + "step": 3000 + }, + { + "entropy": 0.18990286618471144, + "epoch": 7.332129963898917, + "grad_norm": 0.2729308307170868, + "learning_rate": 8.115058651136956e-05, + "loss": 0.11597777366638183, + "mean_token_accuracy": 0.9613875389099121, + "num_tokens": 4298066.0, + "step": 3050 + }, + { + "entropy": 0.18902748223394156, + "epoch": 7.452466907340553, + "grad_norm": 0.3057991564273834, + "learning_rate": 7.449047400700742e-05, + "loss": 0.11589047431945801, + "mean_token_accuracy": 0.9611530381441117, + "num_tokens": 4370032.0, + "step": 3100 + }, + { + "entropy": 0.18580251831561326, + "epoch": 7.57280385078219, + "grad_norm": 0.3325570523738861, + "learning_rate": 6.805281303838264e-05, + "loss": 0.11706151008605957, + "mean_token_accuracy": 0.9613592675328255, + "num_tokens": 4440775.0, + "step": 3150 + }, + { + "entropy": 0.1917814151570201, + "epoch": 7.693140794223827, + "grad_norm": 0.2866221070289612, + "learning_rate": 6.184893367227762e-05, + "loss": 0.11704328536987305, + "mean_token_accuracy": 0.9623443368077278, + "num_tokens": 4512445.0, + "step": 3200 + }, + { + "entropy": 0.1835070162639022, + "epoch": 7.813477737665464, + "grad_norm": 0.4219290018081665, + "learning_rate": 5.5889754527717426e-05, + "loss": 0.11752543449401856, + "mean_token_accuracy": 0.9615215319395065, + "num_tokens": 4582939.0, + "step": 3250 + }, + { + "entropy": 0.19630121663212777, + "epoch": 7.9338146811070995, + "grad_norm": 0.20138691365718842, + "learning_rate": 5.018576355956703e-05, + "loss": 0.11969227790832519, + "mean_token_accuracy": 0.9607423830032349, + "num_tokens": 4651557.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.22974008284037625, + "eval_loss": 1.002966284751892, + "eval_mean_token_accuracy": 0.8310878254190276, + "eval_num_tokens": 4689712.0, + "eval_runtime": 97.8527, + "eval_samples_per_second": 12.907, + "eval_steps_per_second": 1.615, + "step": 3328 + }, + { + "entropy": 0.18697055451797717, + "epoch": 8.05294825511432, + "grad_norm": 0.17605452239513397, + "learning_rate": 4.474699960008424e-05, + "loss": 0.11182424545288086, + "mean_token_accuracy": 0.9623116381240614, + "num_tokens": 4721916.0, + "step": 3350 + }, + { + "entropy": 0.1714377211779356, + "epoch": 8.173285198555957, + "grad_norm": 0.13373211026191711, + "learning_rate": 3.958303469091194e-05, + "loss": 0.10396961212158203, + "mean_token_accuracy": 0.9654406791925431, + "num_tokens": 4791261.0, + "step": 3400 + }, + { + "entropy": 0.17083973079919815, + "epoch": 8.293622141997593, + "grad_norm": 0.1479639858007431, + "learning_rate": 3.470295723660699e-05, + "loss": 0.10398404121398926, + "mean_token_accuracy": 0.9651612070202827, + "num_tokens": 4860316.0, + "step": 3450 + }, + { + "entropy": 0.1823022047057748, + "epoch": 8.41395908543923, + "grad_norm": 0.16216318309307098, + "learning_rate": 3.011535600935398e-05, + "loss": 0.10715325355529785, + "mean_token_accuracy": 0.9631140816211701, + "num_tokens": 4928346.0, + "step": 3500 + }, + { + "entropy": 0.16444372795522214, + "epoch": 8.534296028880867, + "grad_norm": 0.16531634330749512, + "learning_rate": 2.5828305033014883e-05, + "loss": 0.09897437095642089, + "mean_token_accuracy": 0.9659266674518585, + "num_tokens": 5002213.0, + "step": 3550 + }, + { + "entropy": 0.17314248468726873, + "epoch": 8.654632972322503, + "grad_norm": 0.17636360228061676, + "learning_rate": 2.1849349373118615e-05, + "loss": 0.10351550102233886, + "mean_token_accuracy": 0.9645318695902825, + "num_tokens": 5074008.0, + "step": 3600 + }, + { + "entropy": 0.16404723081737757, + "epoch": 8.774969915764139, + "grad_norm": 0.23865492641925812, + "learning_rate": 1.8185491857799596e-05, + "loss": 0.10155345916748047, + "mean_token_accuracy": 0.9654611736536026, + "num_tokens": 5145997.0, + "step": 3650 + }, + { + "entropy": 0.17373924318701028, + "epoch": 8.895306859205776, + "grad_norm": 0.16738754510879517, + "learning_rate": 1.4843180753055772e-05, + "loss": 0.10436760902404785, + "mean_token_accuracy": 0.9643743595480919, + "num_tokens": 5215661.0, + "step": 3700 + }, + { + "epoch": 9.0, + "eval_entropy": 0.21024956582467766, + "eval_loss": 1.0845497846603394, + "eval_mean_token_accuracy": 0.8320421716080436, + "eval_num_tokens": 5275926.0, + "eval_runtime": 97.6962, + "eval_samples_per_second": 12.928, + "eval_steps_per_second": 1.617, + "step": 3744 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9505703716406067e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e11e63a0af28f573d2eeebeee209910debb48760 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-416/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 416, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1697834714208256e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ec0295e1591b530a4e1ca5fd58e5d7fddb00bb50 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-4160/trainer_state.json @@ -0,0 +1,974 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + }, + { + "entropy": 0.6247954857770843, + "epoch": 2.0433212996389893, + "grad_norm": 0.6657149195671082, + "learning_rate": 0.00038865746083879165, + "loss": 0.566479606628418, + "mean_token_accuracy": 0.8411384297139717, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5768365310132504, + "epoch": 2.1636582430806257, + "grad_norm": 0.8292215466499329, + "learning_rate": 0.00038549782182756997, + "loss": 0.5056516265869141, + "mean_token_accuracy": 0.8520097219944001, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5581371414661408, + "epoch": 2.2839951865222625, + "grad_norm": 0.6516403555870056, + "learning_rate": 0.0003820132718802489, + "loss": 0.5099326324462891, + "mean_token_accuracy": 0.8493803012371063, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5893278759717941, + "epoch": 2.404332129963899, + "grad_norm": 0.9996955990791321, + "learning_rate": 0.0003782099436875375, + "loss": 0.5281977462768555, + "mean_token_accuracy": 0.8475887528061867, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.56315244987607, + "epoch": 2.5246690734055353, + "grad_norm": 0.7938012480735779, + "learning_rate": 0.0003740945309791348, + "loss": 0.5179048156738282, + "mean_token_accuracy": 0.8493809035420418, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5777775266766548, + "epoch": 2.645006016847172, + "grad_norm": 0.6226617693901062, + "learning_rate": 0.00036967427674299117, + "loss": 0.530558967590332, + "mean_token_accuracy": 0.8470632100105285, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5784990048408508, + "epoch": 2.765342960288809, + "grad_norm": 0.7198805212974548, + "learning_rate": 0.0003649569604778926, + "loss": 0.5288610458374023, + "mean_token_accuracy": 0.849225849211216, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5914637613296508, + "epoch": 2.8856799037304453, + "grad_norm": 0.5952224135398865, + "learning_rate": 0.000359950884501805, + "loss": 0.5239838790893555, + "mean_token_accuracy": 0.8495543846487998, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5663371389802498, + "eval_loss": 0.6285511255264282, + "eval_mean_token_accuracy": 0.824168049459216, + "eval_num_tokens": 1758642.0, + "eval_runtime": 97.7624, + "eval_samples_per_second": 12.919, + "eval_steps_per_second": 1.616, + "step": 1248 + }, + { + "entropy": 0.5990384999248717, + "epoch": 3.0048134777376654, + "grad_norm": 1.0650150775909424, + "learning_rate": 0.00035466485934007336, + "loss": 0.5286240005493164, + "mean_token_accuracy": 0.8491325546996762, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.469620827883482, + "epoch": 3.125150421179302, + "grad_norm": 0.7563692331314087, + "learning_rate": 0.0003491081882191937, + "loss": 0.4175784301757812, + "mean_token_accuracy": 0.8712207105755806, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4950093144178391, + "epoch": 3.2454873646209386, + "grad_norm": 0.6126220226287842, + "learning_rate": 0.00034329065069344753, + "loss": 0.43592708587646484, + "mean_token_accuracy": 0.8674660983681679, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.48454256035387516, + "epoch": 3.3658243080625754, + "grad_norm": 0.5763334035873413, + "learning_rate": 0.0003372224854332147, + "loss": 0.43666534423828124, + "mean_token_accuracy": 0.8677658352255821, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.49176346719264985, + "epoch": 3.4861612515042117, + "grad_norm": 0.6427484750747681, + "learning_rate": 0.00033091437220525845, + "loss": 0.4353389358520508, + "mean_token_accuracy": 0.8665757381916046, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.47478089153766634, + "epoch": 3.606498194945848, + "grad_norm": 0.7839226126670837, + "learning_rate": 0.0003243774130766948, + "loss": 0.42707630157470705, + "mean_token_accuracy": 0.8689834186434746, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.48098948851227763, + "epoch": 3.726835138387485, + "grad_norm": 0.5966774821281433, + "learning_rate": 0.0003176231128757279, + "loss": 0.4326316452026367, + "mean_token_accuracy": 0.8684966510534287, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4556081013381481, + "epoch": 3.8471720818291217, + "grad_norm": 0.6769846081733704, + "learning_rate": 0.0003106633589435393, + "loss": 0.4301485824584961, + "mean_token_accuracy": 0.868788774907589, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.47818533539772035, + "epoch": 3.967509025270758, + "grad_norm": 0.6685031652450562, + "learning_rate": 0.0003035104002129675, + "loss": 0.43968441009521486, + "mean_token_accuracy": 0.8664855718612671, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.4600734844615188, + "eval_loss": 0.6554375886917114, + "eval_mean_token_accuracy": 0.8273950913284398, + "eval_num_tokens": 2344856.0, + "eval_runtime": 97.4191, + "eval_samples_per_second": 12.965, + "eval_steps_per_second": 1.622, + "step": 1664 + }, + { + "entropy": 0.408212879089394, + "epoch": 4.086642599277979, + "grad_norm": 0.5222322344779968, + "learning_rate": 0.0002961768256507977, + "loss": 0.34373237609863283, + "mean_token_accuracy": 0.892645409311911, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.36320762082934377, + "epoch": 4.206979542719615, + "grad_norm": 0.5896300673484802, + "learning_rate": 0.00028867554210160423, + "loss": 0.3111457633972168, + "mean_token_accuracy": 0.8987097987532615, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.38906625136733053, + "epoch": 4.327316486161251, + "grad_norm": 0.5566349625587463, + "learning_rate": 0.00028101975157213767, + "loss": 0.3163478469848633, + "mean_token_accuracy": 0.8980681902170181, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.4033931617438793, + "epoch": 4.447653429602888, + "grad_norm": 0.485627144575119, + "learning_rate": 0.0002732229279962374, + "loss": 0.3242201232910156, + "mean_token_accuracy": 0.8955107414722443, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.41552690997719766, + "epoch": 4.567990373044525, + "grad_norm": 0.7070091962814331, + "learning_rate": 0.00026529879352116086, + "loss": 0.32594871520996094, + "mean_token_accuracy": 0.8947343465685844, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.40075999580323696, + "epoch": 4.688327316486161, + "grad_norm": 0.6225956082344055, + "learning_rate": 0.00025726129435706634, + "loss": 0.32806293487548827, + "mean_token_accuracy": 0.8947074204683304, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.4083924476802349, + "epoch": 4.808664259927798, + "grad_norm": 0.5319440364837646, + "learning_rate": 0.00024912457623215177, + "loss": 0.33675113677978513, + "mean_token_accuracy": 0.8914315050840378, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.42457151383161545, + "epoch": 4.929001203369435, + "grad_norm": 0.5352131128311157, + "learning_rate": 0.00024090295949664908, + "loss": 0.33766315460205076, + "mean_token_accuracy": 0.891443462073803, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.4211015176923969, + "eval_loss": 0.6591429114341736, + "eval_mean_token_accuracy": 0.8325904256935361, + "eval_num_tokens": 2931070.0, + "eval_runtime": 97.8985, + "eval_samples_per_second": 12.901, + "eval_steps_per_second": 1.614, + "step": 2080 + }, + { + "entropy": 0.37054963635675836, + "epoch": 5.048134777376655, + "grad_norm": 0.9011105298995972, + "learning_rate": 0.00023261091391948947, + "loss": 0.28170028686523435, + "mean_token_accuracy": 0.9086048115383495, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.2936194147914648, + "epoch": 5.1684717208182915, + "grad_norm": 0.5512287616729736, + "learning_rate": 0.00022426303322199643, + "loss": 0.21786537170410156, + "mean_token_accuracy": 0.9288332989811897, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.2911713405698538, + "epoch": 5.2888086642599275, + "grad_norm": 0.5876255631446838, + "learning_rate": 0.00021587400939342754, + "loss": 0.2193107795715332, + "mean_token_accuracy": 0.9280375191569328, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.2986573086678982, + "epoch": 5.409145607701564, + "grad_norm": 0.46124088764190674, + "learning_rate": 0.00020745860683356717, + "loss": 0.22520872116088866, + "mean_token_accuracy": 0.9257059851288796, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.30760848000645635, + "epoch": 5.529482551143201, + "grad_norm": 0.44583287835121155, + "learning_rate": 0.00019903163636787952, + "loss": 0.22967565536499024, + "mean_token_accuracy": 0.9240611773729325, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.3116707868874073, + "epoch": 5.649819494584838, + "grad_norm": 0.4463329613208771, + "learning_rate": 0.00019060792918095394, + "loss": 0.22934612274169922, + "mean_token_accuracy": 0.9238958007097244, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.30317666091024875, + "epoch": 5.770156438026474, + "grad_norm": 0.5684349536895752, + "learning_rate": 0.00018220231071411927, + "loss": 0.22462085723876954, + "mean_token_accuracy": 0.9247557842731475, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.3070871848613024, + "epoch": 5.890493381468111, + "grad_norm": 0.4147462248802185, + "learning_rate": 0.00017382957457316484, + "loss": 0.22498483657836915, + "mean_token_accuracy": 0.9251977476477623, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.315427861636198, + "eval_loss": 0.772193193435669, + "eval_mean_token_accuracy": 0.8272767685636689, + "eval_num_tokens": 3517284.0, + "eval_runtime": 97.8925, + "eval_samples_per_second": 12.902, + "eval_steps_per_second": 1.614, + "step": 2496 + }, + { + "entropy": 0.3059864859237815, + "epoch": 6.009626955475331, + "grad_norm": 0.2763276696205139, + "learning_rate": 0.00016550445649209263, + "loss": 0.21960899353027344, + "mean_token_accuracy": 0.9276399949584344, + "num_tokens": 3523144.0, + "step": 2500 + }, + { + "entropy": 0.22258748069405557, + "epoch": 6.129963898916968, + "grad_norm": 0.42732444405555725, + "learning_rate": 0.00015724160839872102, + "loss": 0.14948601722717286, + "mean_token_accuracy": 0.9504047948122024, + "num_tokens": 3594122.0, + "step": 2550 + }, + { + "entropy": 0.24060228884220122, + "epoch": 6.250300842358604, + "grad_norm": 0.473332941532135, + "learning_rate": 0.000149055572627784, + "loss": 0.15332001686096192, + "mean_token_accuracy": 0.9503934115171433, + "num_tokens": 3663655.0, + "step": 2600 + }, + { + "entropy": 0.2281983983516693, + "epoch": 6.37063778580024, + "grad_norm": 0.4745512306690216, + "learning_rate": 0.00014096075632691189, + "loss": 0.15419836044311525, + "mean_token_accuracy": 0.9501351490616798, + "num_tokens": 3733302.0, + "step": 2650 + }, + { + "entropy": 0.2444649949669838, + "epoch": 6.490974729241877, + "grad_norm": 0.38537177443504333, + "learning_rate": 0.0001329714061005355, + "loss": 0.15538623809814453, + "mean_token_accuracy": 0.9505197313427926, + "num_tokens": 3803551.0, + "step": 2700 + }, + { + "entropy": 0.2471274556219578, + "epoch": 6.611311672683514, + "grad_norm": 0.4447166323661804, + "learning_rate": 0.000125101582936342, + "loss": 0.15631080627441407, + "mean_token_accuracy": 0.9499771365523338, + "num_tokens": 3873200.0, + "step": 2750 + }, + { + "entropy": 0.23467195242643357, + "epoch": 6.731648616125151, + "grad_norm": 0.5019384622573853, + "learning_rate": 0.00011736513745840905, + "loss": 0.1504099941253662, + "mean_token_accuracy": 0.9516549646854401, + "num_tokens": 3943741.0, + "step": 2800 + }, + { + "entropy": 0.2275939880311489, + "epoch": 6.851985559566787, + "grad_norm": 0.45045986771583557, + "learning_rate": 0.00010977568555057236, + "loss": 0.15092979431152342, + "mean_token_accuracy": 0.9501178854703903, + "num_tokens": 4016857.0, + "step": 2850 + }, + { + "entropy": 0.23294402249157428, + "epoch": 6.9723225030084235, + "grad_norm": 0.3174738585948944, + "learning_rate": 0.00010234658439292866, + "loss": 0.15235508918762208, + "mean_token_accuracy": 0.9500771847367286, + "num_tokens": 4087868.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.272654819620561, + "eval_loss": 0.8500064015388489, + "eval_mean_token_accuracy": 0.8321780105934867, + "eval_num_tokens": 4103498.0, + "eval_runtime": 98.0906, + "eval_samples_per_second": 12.876, + "eval_steps_per_second": 1.611, + "step": 2912 + }, + { + "entropy": 0.21614460315969256, + "epoch": 7.091456077015644, + "grad_norm": 0.23043400049209595, + "learning_rate": 9.50909089536471e-05, + "loss": 0.12937673568725586, + "mean_token_accuracy": 0.9584822886519961, + "num_tokens": 4154257.0, + "step": 2950 + }, + { + "entropy": 0.19473189041018485, + "epoch": 7.21179302045728, + "grad_norm": 0.27521875500679016, + "learning_rate": 8.802142897746567e-05, + "loss": 0.11380335807800293, + "mean_token_accuracy": 0.9625545349717141, + "num_tokens": 4226995.0, + "step": 3000 + }, + { + "entropy": 0.18990286618471144, + "epoch": 7.332129963898917, + "grad_norm": 0.2729308307170868, + "learning_rate": 8.115058651136956e-05, + "loss": 0.11597777366638183, + "mean_token_accuracy": 0.9613875389099121, + "num_tokens": 4298066.0, + "step": 3050 + }, + { + "entropy": 0.18902748223394156, + "epoch": 7.452466907340553, + "grad_norm": 0.3057991564273834, + "learning_rate": 7.449047400700742e-05, + "loss": 0.11589047431945801, + "mean_token_accuracy": 0.9611530381441117, + "num_tokens": 4370032.0, + "step": 3100 + }, + { + "entropy": 0.18580251831561326, + "epoch": 7.57280385078219, + "grad_norm": 0.3325570523738861, + "learning_rate": 6.805281303838264e-05, + "loss": 0.11706151008605957, + "mean_token_accuracy": 0.9613592675328255, + "num_tokens": 4440775.0, + "step": 3150 + }, + { + "entropy": 0.1917814151570201, + "epoch": 7.693140794223827, + "grad_norm": 0.2866221070289612, + "learning_rate": 6.184893367227762e-05, + "loss": 0.11704328536987305, + "mean_token_accuracy": 0.9623443368077278, + "num_tokens": 4512445.0, + "step": 3200 + }, + { + "entropy": 0.1835070162639022, + "epoch": 7.813477737665464, + "grad_norm": 0.4219290018081665, + "learning_rate": 5.5889754527717426e-05, + "loss": 0.11752543449401856, + "mean_token_accuracy": 0.9615215319395065, + "num_tokens": 4582939.0, + "step": 3250 + }, + { + "entropy": 0.19630121663212777, + "epoch": 7.9338146811070995, + "grad_norm": 0.20138691365718842, + "learning_rate": 5.018576355956703e-05, + "loss": 0.11969227790832519, + "mean_token_accuracy": 0.9607423830032349, + "num_tokens": 4651557.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.22974008284037625, + "eval_loss": 1.002966284751892, + "eval_mean_token_accuracy": 0.8310878254190276, + "eval_num_tokens": 4689712.0, + "eval_runtime": 97.8527, + "eval_samples_per_second": 12.907, + "eval_steps_per_second": 1.615, + "step": 3328 + }, + { + "entropy": 0.18697055451797717, + "epoch": 8.05294825511432, + "grad_norm": 0.17605452239513397, + "learning_rate": 4.474699960008424e-05, + "loss": 0.11182424545288086, + "mean_token_accuracy": 0.9623116381240614, + "num_tokens": 4721916.0, + "step": 3350 + }, + { + "entropy": 0.1714377211779356, + "epoch": 8.173285198555957, + "grad_norm": 0.13373211026191711, + "learning_rate": 3.958303469091194e-05, + "loss": 0.10396961212158203, + "mean_token_accuracy": 0.9654406791925431, + "num_tokens": 4791261.0, + "step": 3400 + }, + { + "entropy": 0.17083973079919815, + "epoch": 8.293622141997593, + "grad_norm": 0.1479639858007431, + "learning_rate": 3.470295723660699e-05, + "loss": 0.10398404121398926, + "mean_token_accuracy": 0.9651612070202827, + "num_tokens": 4860316.0, + "step": 3450 + }, + { + "entropy": 0.1823022047057748, + "epoch": 8.41395908543923, + "grad_norm": 0.16216318309307098, + "learning_rate": 3.011535600935398e-05, + "loss": 0.10715325355529785, + "mean_token_accuracy": 0.9631140816211701, + "num_tokens": 4928346.0, + "step": 3500 + }, + { + "entropy": 0.16444372795522214, + "epoch": 8.534296028880867, + "grad_norm": 0.16531634330749512, + "learning_rate": 2.5828305033014883e-05, + "loss": 0.09897437095642089, + "mean_token_accuracy": 0.9659266674518585, + "num_tokens": 5002213.0, + "step": 3550 + }, + { + "entropy": 0.17314248468726873, + "epoch": 8.654632972322503, + "grad_norm": 0.17636360228061676, + "learning_rate": 2.1849349373118615e-05, + "loss": 0.10351550102233886, + "mean_token_accuracy": 0.9645318695902825, + "num_tokens": 5074008.0, + "step": 3600 + }, + { + "entropy": 0.16404723081737757, + "epoch": 8.774969915764139, + "grad_norm": 0.23865492641925812, + "learning_rate": 1.8185491857799596e-05, + "loss": 0.10155345916748047, + "mean_token_accuracy": 0.9654611736536026, + "num_tokens": 5145997.0, + "step": 3650 + }, + { + "entropy": 0.17373924318701028, + "epoch": 8.895306859205776, + "grad_norm": 0.16738754510879517, + "learning_rate": 1.4843180753055772e-05, + "loss": 0.10436760902404785, + "mean_token_accuracy": 0.9643743595480919, + "num_tokens": 5215661.0, + "step": 3700 + }, + { + "epoch": 9.0, + "eval_entropy": 0.21024956582467766, + "eval_loss": 1.0845497846603394, + "eval_mean_token_accuracy": 0.8320421716080436, + "eval_num_tokens": 5275926.0, + "eval_runtime": 97.6962, + "eval_samples_per_second": 12.928, + "eval_steps_per_second": 1.617, + "step": 3744 + }, + { + "entropy": 0.17217807543247637, + "epoch": 9.014440433212997, + "grad_norm": 0.12622694671154022, + "learning_rate": 1.1828298414017183e-05, + "loss": 0.10267382621765136, + "mean_token_accuracy": 0.9648220102594356, + "num_tokens": 5284952.0, + "step": 3750 + }, + { + "entropy": 0.1644303062185645, + "epoch": 9.134777376654633, + "grad_norm": 0.12117180228233337, + "learning_rate": 9.146150932198844e-06, + "loss": 0.09394286155700683, + "mean_token_accuracy": 0.9678797456622124, + "num_tokens": 5357462.0, + "step": 3800 + }, + { + "entropy": 0.16291595183312893, + "epoch": 9.25511432009627, + "grad_norm": 0.15647734701633453, + "learning_rate": 6.801458796958289e-06, + "loss": 0.09422736167907715, + "mean_token_accuracy": 0.9676002004742622, + "num_tokens": 5429842.0, + "step": 3850 + }, + { + "entropy": 0.16274009495973588, + "epoch": 9.375451263537906, + "grad_norm": 0.17268836498260498, + "learning_rate": 4.798348587592805e-06, + "loss": 0.09515288352966308, + "mean_token_accuracy": 0.9668962359428406, + "num_tokens": 5501125.0, + "step": 3900 + }, + { + "entropy": 0.16325047258287667, + "epoch": 9.495788206979542, + "grad_norm": 0.12881265580654144, + "learning_rate": 3.1403457106989303e-06, + "loss": 0.09702346801757812, + "mean_token_accuracy": 0.9665210220217705, + "num_tokens": 5571188.0, + "step": 3950 + }, + { + "entropy": 0.1654203213751316, + "epoch": 9.61612515042118, + "grad_norm": 0.15134739875793457, + "learning_rate": 1.8303681955751818e-06, + "loss": 0.09895826339721679, + "mean_token_accuracy": 0.966290439069271, + "num_tokens": 5640335.0, + "step": 4000 + }, + { + "entropy": 0.16204755686223507, + "epoch": 9.736462093862816, + "grad_norm": 0.18626996874809265, + "learning_rate": 8.707215585888064e-07, + "loss": 0.09639920234680176, + "mean_token_accuracy": 0.9664874693751335, + "num_tokens": 5711388.0, + "step": 4050 + }, + { + "entropy": 0.17234525412321092, + "epoch": 9.856799037304452, + "grad_norm": 0.1622859239578247, + "learning_rate": 2.6309474554443897e-07, + "loss": 0.10020747184753417, + "mean_token_accuracy": 0.9652139312028885, + "num_tokens": 5779733.0, + "step": 4100 + }, + { + "entropy": 0.16553655911237, + "epoch": 9.97713598074609, + "grad_norm": 0.1983521580696106, + "learning_rate": 8.557159196288404e-09, + "loss": 0.09647385597229004, + "mean_token_accuracy": 0.9671455720067024, + "num_tokens": 5850073.0, + "step": 4150 + }, + { + "epoch": 10.0, + "eval_entropy": 0.20790981661669816, + "eval_loss": 1.132350206375122, + "eval_mean_token_accuracy": 0.8318726111061966, + "eval_num_tokens": 5862140.0, + "eval_runtime": 97.9118, + "eval_samples_per_second": 12.899, + "eval_steps_per_second": 1.614, + "step": 4160 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.166952051721257e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..7f4fd4e35c164ec91a6d4c5e235f4c4583405012 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.013737379760196178, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e2bf642262585054d2caa3c341c9c9b7ea965d26 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/checkpoint-832/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 832, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.5280762761831284, + "epoch": 0.12033694344163658, + "grad_norm": 0.9672366976737976, + "learning_rate": 4.732406041160677e-05, + "loss": 1.348579864501953, + "mean_token_accuracy": 0.7104641751945019, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7766477742791176, + "epoch": 0.24067388688327315, + "grad_norm": 0.6913937330245972, + "learning_rate": 9.561391797447082e-05, + "loss": 0.6702117919921875, + "mean_token_accuracy": 0.818458753824234, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.740069375038147, + "epoch": 0.36101083032490977, + "grad_norm": 0.6093149185180664, + "learning_rate": 0.0001439037755373349, + "loss": 0.6354790878295898, + "mean_token_accuracy": 0.8267712336778641, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.7337970679998398, + "epoch": 0.4813477737665463, + "grad_norm": 0.8280653357505798, + "learning_rate": 0.00019219363310019896, + "loss": 0.6319454193115235, + "mean_token_accuracy": 0.8252587172389031, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.716901783645153, + "epoch": 0.601684717208183, + "grad_norm": 0.920710563659668, + "learning_rate": 0.000240483490663063, + "loss": 0.6128461456298828, + "mean_token_accuracy": 0.8311672276258468, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7144344159960747, + "epoch": 0.7220216606498195, + "grad_norm": 0.7012578248977661, + "learning_rate": 0.000288773348225927, + "loss": 0.6241677856445312, + "mean_token_accuracy": 0.828495902121067, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7210446110367775, + "epoch": 0.8423586040914561, + "grad_norm": 0.6508389115333557, + "learning_rate": 0.0003370632057887911, + "loss": 0.6353719329833984, + "mean_token_accuracy": 0.8281483414769173, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.7273332607746125, + "epoch": 0.9626955475330926, + "grad_norm": 1.0850260257720947, + "learning_rate": 0.00038535306335165515, + "loss": 0.639124755859375, + "mean_token_accuracy": 0.8248243638873101, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6947960517828977, + "eval_loss": 0.6629942059516907, + "eval_mean_token_accuracy": 0.8138390719136105, + "eval_num_tokens": 586214.0, + "eval_runtime": 98.4976, + "eval_samples_per_second": 12.823, + "eval_steps_per_second": 1.604, + "step": 416 + }, + { + "entropy": 0.7191893578779818, + "epoch": 1.0818291215403129, + "grad_norm": 0.9315849542617798, + "learning_rate": 0.0004016946048643255, + "loss": 0.6389211273193359, + "mean_token_accuracy": 0.8279516528351138, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.7125428369641305, + "epoch": 1.2021660649819494, + "grad_norm": 1.3237719535827637, + "learning_rate": 0.0004012846160482895, + "loss": 0.6376816558837891, + "mean_token_accuracy": 0.8263090917468071, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6965643188357353, + "epoch": 1.322503008423586, + "grad_norm": 1.056685447692871, + "learning_rate": 0.0004005219320692939, + "loss": 0.6283835601806641, + "mean_token_accuracy": 0.8268012097477913, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.7037768799066544, + "epoch": 1.4428399518652226, + "grad_norm": 0.8307286500930786, + "learning_rate": 0.0003994078952254899, + "loss": 0.6389706802368164, + "mean_token_accuracy": 0.8249652540683746, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6932012620568275, + "epoch": 1.5631768953068592, + "grad_norm": 1.3095203638076782, + "learning_rate": 0.0003979444661842533, + "loss": 0.6213974380493164, + "mean_token_accuracy": 0.8282416883111, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6818545503914356, + "epoch": 1.6835138387484958, + "grad_norm": 0.753699541091919, + "learning_rate": 0.0003961342205314764, + "loss": 0.6077573013305664, + "mean_token_accuracy": 0.833371383547783, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6607842321693898, + "epoch": 1.8038507821901324, + "grad_norm": 0.6580342054367065, + "learning_rate": 0.00039398034423862257, + "loss": 0.6033860778808594, + "mean_token_accuracy": 0.8333631888031959, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6716558849811554, + "epoch": 1.924187725631769, + "grad_norm": 0.6773555278778076, + "learning_rate": 0.00039148662805552455, + "loss": 0.6086238861083985, + "mean_token_accuracy": 0.8315111309289932, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6449016423165044, + "eval_loss": 0.637292206287384, + "eval_mean_token_accuracy": 0.8189603724057162, + "eval_num_tokens": 1172428.0, + "eval_runtime": 98.1797, + "eval_samples_per_second": 12.864, + "eval_steps_per_second": 1.609, + "step": 832 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.324635680612352e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbca7a97b9e49d40f2cdb3ba39ccfb08bb0e1018 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07104296128667408, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..49462b1fa66ca1d83f580307cd21bde1a804f3cf --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-416/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 416, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.621582981646061, + "epoch": 0.12033694344163658, + "grad_norm": 1.7840632200241089, + "learning_rate": 3.557416011989973e-05, + "loss": 1.4308599853515624, + "mean_token_accuracy": 0.6972626076638698, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7890771624445915, + "epoch": 0.24067388688327315, + "grad_norm": 1.1691383123397827, + "learning_rate": 7.187432350755252e-05, + "loss": 0.68025390625, + "mean_token_accuracy": 0.8160018160939216, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.7458965721726417, + "epoch": 0.36101083032490977, + "grad_norm": 0.9552986025810242, + "learning_rate": 0.0001081744868952053, + "loss": 0.6382268524169922, + "mean_token_accuracy": 0.8264451423287391, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.735072555243969, + "epoch": 0.4813477737665463, + "grad_norm": 1.0810377597808838, + "learning_rate": 0.0001444746502828581, + "loss": 0.6294943237304688, + "mean_token_accuracy": 0.8261765521764756, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.7114953261613846, + "epoch": 0.601684717208183, + "grad_norm": 0.9742010831832886, + "learning_rate": 0.00018077481367051088, + "loss": 0.602623176574707, + "mean_token_accuracy": 0.83393067330122, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7069582852721215, + "epoch": 0.7220216606498195, + "grad_norm": 0.7885417342185974, + "learning_rate": 0.00021707497705816367, + "loss": 0.6058937072753906, + "mean_token_accuracy": 0.8321960058808326, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7101716002821923, + "epoch": 0.8423586040914561, + "grad_norm": 0.8627666234970093, + "learning_rate": 0.00025337514044581645, + "loss": 0.6131188583374023, + "mean_token_accuracy": 0.8330579137802124, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.6990390625596047, + "epoch": 0.9626955475330926, + "grad_norm": 1.1322276592254639, + "learning_rate": 0.00028967530383346927, + "loss": 0.6065136337280274, + "mean_token_accuracy": 0.8326930233836174, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6208525450923775, + "eval_loss": 0.6319083571434021, + "eval_mean_token_accuracy": 0.8220708449430103, + "eval_num_tokens": 586214.0, + "eval_runtime": 93.9256, + "eval_samples_per_second": 13.436, + "eval_steps_per_second": 1.682, + "step": 416 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.1205070849316864e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbca7a97b9e49d40f2cdb3ba39ccfb08bb0e1018 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07104296128667408, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..96820823bfce8cb119f8cd3fd33591fda5d55a29 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-4160/trainer_state.json @@ -0,0 +1,974 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.621582981646061, + "epoch": 0.12033694344163658, + "grad_norm": 1.7840632200241089, + "learning_rate": 3.557416011989973e-05, + "loss": 1.4308599853515624, + "mean_token_accuracy": 0.6972626076638698, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7890771624445915, + "epoch": 0.24067388688327315, + "grad_norm": 1.1691383123397827, + "learning_rate": 7.187432350755252e-05, + "loss": 0.68025390625, + "mean_token_accuracy": 0.8160018160939216, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.7458965721726417, + "epoch": 0.36101083032490977, + "grad_norm": 0.9552986025810242, + "learning_rate": 0.0001081744868952053, + "loss": 0.6382268524169922, + "mean_token_accuracy": 0.8264451423287391, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.735072555243969, + "epoch": 0.4813477737665463, + "grad_norm": 1.0810377597808838, + "learning_rate": 0.0001444746502828581, + "loss": 0.6294943237304688, + "mean_token_accuracy": 0.8261765521764756, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.7114953261613846, + "epoch": 0.601684717208183, + "grad_norm": 0.9742010831832886, + "learning_rate": 0.00018077481367051088, + "loss": 0.602623176574707, + "mean_token_accuracy": 0.83393067330122, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7069582852721215, + "epoch": 0.7220216606498195, + "grad_norm": 0.7885417342185974, + "learning_rate": 0.00021707497705816367, + "loss": 0.6058937072753906, + "mean_token_accuracy": 0.8321960058808326, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7101716002821923, + "epoch": 0.8423586040914561, + "grad_norm": 0.8627666234970093, + "learning_rate": 0.00025337514044581645, + "loss": 0.6131188583374023, + "mean_token_accuracy": 0.8330579137802124, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.6990390625596047, + "epoch": 0.9626955475330926, + "grad_norm": 1.1322276592254639, + "learning_rate": 0.00028967530383346927, + "loss": 0.6065136337280274, + "mean_token_accuracy": 0.8326930233836174, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6208525450923775, + "eval_loss": 0.6319083571434021, + "eval_mean_token_accuracy": 0.8220708449430103, + "eval_num_tokens": 586214.0, + "eval_runtime": 93.9256, + "eval_samples_per_second": 13.436, + "eval_steps_per_second": 1.682, + "step": 416 + }, + { + "entropy": 0.6688173970188758, + "epoch": 1.0818291215403129, + "grad_norm": 1.2222388982772827, + "learning_rate": 0.0003019594698437709, + "loss": 0.5917711639404297, + "mean_token_accuracy": 0.8361045230518688, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.6467562046647072, + "epoch": 1.2021660649819494, + "grad_norm": 1.0695290565490723, + "learning_rate": 0.00030165127549903, + "loss": 0.5744387817382812, + "mean_token_accuracy": 0.8404921460151672, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6470985013246536, + "epoch": 1.322503008423586, + "grad_norm": 1.0010168552398682, + "learning_rate": 0.0003010779552523376, + "loss": 0.574291648864746, + "mean_token_accuracy": 0.8386730599403381, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.6490107670426368, + "epoch": 1.4428399518652226, + "grad_norm": 0.9982119202613831, + "learning_rate": 0.0003002405181280448, + "loss": 0.5834595870971679, + "mean_token_accuracy": 0.8363789242506027, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6380249202251435, + "epoch": 1.5631768953068592, + "grad_norm": 1.0275771617889404, + "learning_rate": 0.00029914043798732445, + "loss": 0.5688864517211915, + "mean_token_accuracy": 0.8405828747153282, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6351437552273274, + "epoch": 1.6835138387484958, + "grad_norm": 0.9777424931526184, + "learning_rate": 0.0002977796509342244, + "loss": 0.5608405303955079, + "mean_token_accuracy": 0.8426707127690315, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6300888039171696, + "epoch": 1.8038507821901324, + "grad_norm": 1.1589821577072144, + "learning_rate": 0.00029616055190818973, + "loss": 0.5640585708618164, + "mean_token_accuracy": 0.8411854481697083, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6372530253231525, + "epoch": 1.924187725631769, + "grad_norm": 1.3437201976776123, + "learning_rate": 0.00029428599046905007, + "loss": 0.5722800445556641, + "mean_token_accuracy": 0.8391004773974419, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5832181644590595, + "eval_loss": 0.5960000157356262, + "eval_mean_token_accuracy": 0.8264509171624727, + "eval_num_tokens": 1172428.0, + "eval_runtime": 93.0484, + "eval_samples_per_second": 13.563, + "eval_steps_per_second": 1.698, + "step": 832 + }, + { + "entropy": 0.5875298981413697, + "epoch": 2.0433212996389893, + "grad_norm": 0.7619560956954956, + "learning_rate": 0.00029215926578188983, + "loss": 0.5282161331176758, + "mean_token_accuracy": 0.8504815243109308, + "num_tokens": 1197927.0, + "step": 850 + }, + { + "entropy": 0.5255681477487087, + "epoch": 2.1636582430806257, + "grad_norm": 0.992935299873352, + "learning_rate": 0.00028978412081062877, + "loss": 0.4643865966796875, + "mean_token_accuracy": 0.8633851459622384, + "num_tokens": 1268402.0, + "step": 900 + }, + { + "entropy": 0.5039383636415005, + "epoch": 2.2839951865222625, + "grad_norm": 0.9263365864753723, + "learning_rate": 0.0002871647357305314, + "loss": 0.4713961791992187, + "mean_token_accuracy": 0.8596943366527557, + "num_tokens": 1339699.0, + "step": 950 + }, + { + "entropy": 0.5475333638489246, + "epoch": 2.404332129963899, + "grad_norm": 0.864780843257904, + "learning_rate": 0.00028430572057123923, + "loss": 0.4895991516113281, + "mean_token_accuracy": 0.8570118647813797, + "num_tokens": 1406119.0, + "step": 1000 + }, + { + "entropy": 0.5374829450249672, + "epoch": 2.5246690734055353, + "grad_norm": 1.2018821239471436, + "learning_rate": 0.0002812121071032731, + "loss": 0.4818190383911133, + "mean_token_accuracy": 0.8579521527886391, + "num_tokens": 1476113.0, + "step": 1050 + }, + { + "entropy": 0.5575080865621567, + "epoch": 2.645006016847172, + "grad_norm": 0.8148619532585144, + "learning_rate": 0.0002778893399822872, + "loss": 0.49133522033691407, + "mean_token_accuracy": 0.8558934706449509, + "num_tokens": 1548921.0, + "step": 1100 + }, + { + "entropy": 0.5533312886953354, + "epoch": 2.765342960288809, + "grad_norm": 0.8920133709907532, + "learning_rate": 0.0002743432671666573, + "loss": 0.48731986999511717, + "mean_token_accuracy": 0.8587287512421607, + "num_tokens": 1619109.0, + "step": 1150 + }, + { + "entropy": 0.5609780684113502, + "epoch": 2.8856799037304453, + "grad_norm": 0.9178280234336853, + "learning_rate": 0.00027058012962527165, + "loss": 0.4861798858642578, + "mean_token_accuracy": 0.858745946586132, + "num_tokens": 1690983.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.537393679158597, + "eval_loss": 0.5679827928543091, + "eval_mean_token_accuracy": 0.8394716966755783, + "eval_num_tokens": 1758642.0, + "eval_runtime": 93.4589, + "eval_samples_per_second": 13.503, + "eval_steps_per_second": 1.691, + "step": 1248 + }, + { + "entropy": 0.5624644063939952, + "epoch": 3.0048134777376654, + "grad_norm": 1.0631294250488281, + "learning_rate": 0.00026660655035363453, + "loss": 0.49134925842285154, + "mean_token_accuracy": 0.8562649900865074, + "num_tokens": 1760928.0, + "step": 1250 + }, + { + "entropy": 0.44835601732134817, + "epoch": 3.125150421179302, + "grad_norm": 1.01238214969635, + "learning_rate": 0.00026242952271761805, + "loss": 0.3744523239135742, + "mean_token_accuracy": 0.88196124792099, + "num_tokens": 1830375.0, + "step": 1300 + }, + { + "entropy": 0.4598517197370529, + "epoch": 3.2454873646209386, + "grad_norm": 0.7782613635063171, + "learning_rate": 0.00025805639814537273, + "loss": 0.39311359405517576, + "mean_token_accuracy": 0.8776173835992813, + "num_tokens": 1897633.0, + "step": 1350 + }, + { + "entropy": 0.4684299013018608, + "epoch": 3.3658243080625754, + "grad_norm": 0.8989675641059875, + "learning_rate": 0.0002534948731890613, + "loss": 0.3908197784423828, + "mean_token_accuracy": 0.8800400498509408, + "num_tokens": 1965963.0, + "step": 1400 + }, + { + "entropy": 0.46587940752506257, + "epoch": 3.4861612515042117, + "grad_norm": 0.8078500628471375, + "learning_rate": 0.0002487529759791859, + "loss": 0.3927593612670898, + "mean_token_accuracy": 0.878325537443161, + "num_tokens": 2038354.0, + "step": 1450 + }, + { + "entropy": 0.44822451800107954, + "epoch": 3.606498194945848, + "grad_norm": 1.041649580001831, + "learning_rate": 0.00024383905209534836, + "loss": 0.3880035400390625, + "mean_token_accuracy": 0.8795277917385101, + "num_tokens": 2111589.0, + "step": 1500 + }, + { + "entropy": 0.44258872836828234, + "epoch": 3.726835138387485, + "grad_norm": 0.774890661239624, + "learning_rate": 0.0002387617498783109, + "loss": 0.39501266479492186, + "mean_token_accuracy": 0.878902695775032, + "num_tokens": 2183548.0, + "step": 1550 + }, + { + "entropy": 0.4352216117084026, + "epoch": 3.8471720818291217, + "grad_norm": 1.009773850440979, + "learning_rate": 0.0002335300052092069, + "loss": 0.39223480224609375, + "mean_token_accuracy": 0.8805220025777817, + "num_tokens": 2256444.0, + "step": 1600 + }, + { + "entropy": 0.4330697976052761, + "epoch": 3.967509025270758, + "grad_norm": 1.016603946685791, + "learning_rate": 0.00022815302578269115, + "loss": 0.4016778564453125, + "mean_token_accuracy": 0.8759643957018852, + "num_tokens": 2326173.0, + "step": 1650 + }, + { + "epoch": 4.0, + "eval_entropy": 0.45692482835884335, + "eval_loss": 0.608729898929596, + "eval_mean_token_accuracy": 0.8357140938692456, + "eval_num_tokens": 2344856.0, + "eval_runtime": 93.0768, + "eval_samples_per_second": 13.559, + "eval_steps_per_second": 1.698, + "step": 1664 + }, + { + "entropy": 0.3643836485165538, + "epoch": 4.086642599277979, + "grad_norm": 0.8249331116676331, + "learning_rate": 0.00022264027490170662, + "loss": 0.3145031547546387, + "mean_token_accuracy": 0.901862360311277, + "num_tokens": 2396595.0, + "step": 1700 + }, + { + "entropy": 0.32612807080149653, + "epoch": 4.206979542719615, + "grad_norm": 0.8062915802001953, + "learning_rate": 0.0002170014548223896, + "loss": 0.28444656372070315, + "mean_token_accuracy": 0.9078416880965233, + "num_tokens": 2466738.0, + "step": 1750 + }, + { + "entropy": 0.3343695695698261, + "epoch": 4.327316486161251, + "grad_norm": 0.7805173993110657, + "learning_rate": 0.0002112464896784254, + "loss": 0.28478408813476563, + "mean_token_accuracy": 0.9088497453927994, + "num_tokens": 2537409.0, + "step": 1800 + }, + { + "entropy": 0.35255078218877317, + "epoch": 4.447653429602888, + "grad_norm": 0.7668603658676147, + "learning_rate": 0.00020538550801490652, + "loss": 0.2920229530334473, + "mean_token_accuracy": 0.9055094057321549, + "num_tokens": 2607327.0, + "step": 1850 + }, + { + "entropy": 0.3559434647858143, + "epoch": 4.567990373044525, + "grad_norm": 0.8114978671073914, + "learning_rate": 0.0001994288249624343, + "loss": 0.29595083236694336, + "mean_token_accuracy": 0.9050057205557823, + "num_tokens": 2677917.0, + "step": 1900 + }, + { + "entropy": 0.3516669125854969, + "epoch": 4.688327316486161, + "grad_norm": 0.8359290957450867, + "learning_rate": 0.00019338692408283579, + "loss": 0.29980892181396485, + "mean_token_accuracy": 0.9040750420093536, + "num_tokens": 2750476.0, + "step": 1950 + }, + { + "entropy": 0.3632228713482618, + "epoch": 4.808664259927798, + "grad_norm": 0.8737912178039551, + "learning_rate": 0.00018727043891844766, + "loss": 0.308858699798584, + "mean_token_accuracy": 0.9016904199123382, + "num_tokens": 2819749.0, + "step": 2000 + }, + { + "entropy": 0.3624253248423338, + "epoch": 4.929001203369435, + "grad_norm": 0.769707202911377, + "learning_rate": 0.00018109013427743917, + "loss": 0.3070379638671875, + "mean_token_accuracy": 0.8996906995773315, + "num_tokens": 2887924.0, + "step": 2050 + }, + { + "epoch": 5.0, + "eval_entropy": 0.3635720859600019, + "eval_loss": 0.6491048336029053, + "eval_mean_token_accuracy": 0.8373204801656022, + "eval_num_tokens": 2931070.0, + "eval_runtime": 93.1917, + "eval_samples_per_second": 13.542, + "eval_steps_per_second": 1.695, + "step": 2080 + }, + { + "entropy": 0.2928671921895008, + "epoch": 5.048134777376655, + "grad_norm": 0.8802099227905273, + "learning_rate": 0.00017485688728811205, + "loss": 0.2584196662902832, + "mean_token_accuracy": 0.9167190061675178, + "num_tokens": 2959123.0, + "step": 2100 + }, + { + "entropy": 0.23706517547369002, + "epoch": 5.1684717208182915, + "grad_norm": 0.8717109560966492, + "learning_rate": 0.0001685816682555203, + "loss": 0.19960372924804687, + "mean_token_accuracy": 0.9359423834085464, + "num_tokens": 3030412.0, + "step": 2150 + }, + { + "entropy": 0.23956211373209954, + "epoch": 5.2888086642599275, + "grad_norm": 0.9274334907531738, + "learning_rate": 0.00016227552135410247, + "loss": 0.20083864212036132, + "mean_token_accuracy": 0.934300790131092, + "num_tokens": 3101921.0, + "step": 2200 + }, + { + "entropy": 0.24641500979661943, + "epoch": 5.409145607701564, + "grad_norm": 0.7631744742393494, + "learning_rate": 0.00015594954519030602, + "loss": 0.205546875, + "mean_token_accuracy": 0.9327731326222419, + "num_tokens": 3170451.0, + "step": 2250 + }, + { + "entropy": 0.25113916628062727, + "epoch": 5.529482551143201, + "grad_norm": 0.6569008827209473, + "learning_rate": 0.00014961487326941325, + "loss": 0.20732742309570312, + "mean_token_accuracy": 0.9321904668211937, + "num_tokens": 3239301.0, + "step": 2300 + }, + { + "entropy": 0.26152016781270504, + "epoch": 5.649819494584838, + "grad_norm": 0.6772239804267883, + "learning_rate": 0.00014328265440094644, + "loss": 0.20642175674438476, + "mean_token_accuracy": 0.9322906348109246, + "num_tokens": 3311023.0, + "step": 2350 + }, + { + "entropy": 0.24740293987095355, + "epoch": 5.770156438026474, + "grad_norm": 0.6547145843505859, + "learning_rate": 0.00013696403307713828, + "loss": 0.20815895080566407, + "mean_token_accuracy": 0.9309536895155907, + "num_tokens": 3381817.0, + "step": 2400 + }, + { + "entropy": 0.2527031201869249, + "epoch": 5.890493381468111, + "grad_norm": 0.6869855523109436, + "learning_rate": 0.00013067012985899996, + "loss": 0.20436925888061525, + "mean_token_accuracy": 0.9333040246367454, + "num_tokens": 3454341.0, + "step": 2450 + }, + { + "epoch": 6.0, + "eval_entropy": 0.2930484226421465, + "eval_loss": 0.7588958740234375, + "eval_mean_token_accuracy": 0.8304676813415334, + "eval_num_tokens": 3517284.0, + "eval_runtime": 93.2875, + "eval_samples_per_second": 13.528, + "eval_steps_per_second": 1.694, + "step": 2496 + }, + { + "entropy": 0.2468054991478872, + "epoch": 6.009626955475331, + "grad_norm": 0.43051984906196594, + "learning_rate": 0.00012441202180450814, + "loss": 0.201207218170166, + "mean_token_accuracy": 0.9344112858025715, + "num_tokens": 3523144.0, + "step": 2500 + }, + { + "entropy": 0.18545001976191997, + "epoch": 6.129963898916968, + "grad_norm": 0.5197061896324158, + "learning_rate": 0.00011820072297335549, + "loss": 0.1436307907104492, + "mean_token_accuracy": 0.9535153642296791, + "num_tokens": 3594122.0, + "step": 2550 + }, + { + "entropy": 0.18873635839670896, + "epoch": 6.250300842358604, + "grad_norm": 0.7676698565483093, + "learning_rate": 0.00011204716504257579, + "loss": 0.14807543754577637, + "mean_token_accuracy": 0.9526472732424736, + "num_tokens": 3663655.0, + "step": 2600 + }, + { + "entropy": 0.18774624072015286, + "epoch": 6.37063778580024, + "grad_norm": 0.8916293978691101, + "learning_rate": 0.0001059621780671604, + "loss": 0.14720329284667968, + "mean_token_accuracy": 0.9518964666128159, + "num_tokens": 3733302.0, + "step": 2650 + }, + { + "entropy": 0.1833176515996456, + "epoch": 6.490974729241877, + "grad_norm": 0.6148649454116821, + "learning_rate": 9.995647141952533e-05, + "loss": 0.14619502067565918, + "mean_token_accuracy": 0.9534434914588928, + "num_tokens": 3803551.0, + "step": 2700 + }, + { + "entropy": 0.1874897350370884, + "epoch": 6.611311672683514, + "grad_norm": 0.632608950138092, + "learning_rate": 9.40406149413764e-05, + "loss": 0.14901174545288087, + "mean_token_accuracy": 0.9529427444934845, + "num_tokens": 3873200.0, + "step": 2750 + }, + { + "entropy": 0.18832745119929314, + "epoch": 6.731648616125151, + "grad_norm": 0.5599837303161621, + "learning_rate": 8.822502034114294e-05, + "loss": 0.1418892288208008, + "mean_token_accuracy": 0.9546939292550087, + "num_tokens": 3943741.0, + "step": 2800 + }, + { + "entropy": 0.19739359058439732, + "epoch": 6.851985559566787, + "grad_norm": 0.520361602306366, + "learning_rate": 8.251992286972134e-05, + "loss": 0.14427796363830567, + "mean_token_accuracy": 0.9530402758717537, + "num_tokens": 4016857.0, + "step": 2850 + }, + { + "entropy": 0.20963006667792797, + "epoch": 6.9723225030084235, + "grad_norm": 0.6935341358184814, + "learning_rate": 7.693536330677795e-05, + "loss": 0.14530478477478026, + "mean_token_accuracy": 0.9525505018234253, + "num_tokens": 4087868.0, + "step": 2900 + }, + { + "epoch": 7.0, + "eval_entropy": 0.25904222511792485, + "eval_loss": 0.8478371500968933, + "eval_mean_token_accuracy": 0.8341704878625991, + "eval_num_tokens": 4103498.0, + "eval_runtime": 93.5189, + "eval_samples_per_second": 13.495, + "eval_steps_per_second": 1.689, + "step": 2912 + }, + { + "entropy": 0.19186869959789093, + "epoch": 7.091456077015644, + "grad_norm": 0.456407755613327, + "learning_rate": 7.148117028931405e-05, + "loss": 0.12616381645202637, + "mean_token_accuracy": 0.9597405806334331, + "num_tokens": 4154257.0, + "step": 2950 + }, + { + "entropy": 0.17192038245499133, + "epoch": 7.21179302045728, + "grad_norm": 0.347380667924881, + "learning_rate": 6.61669430135957e-05, + "loss": 0.11209571838378907, + "mean_token_accuracy": 0.9631491515040398, + "num_tokens": 4226995.0, + "step": 3000 + }, + { + "entropy": 0.16242246367037297, + "epoch": 7.332129963898917, + "grad_norm": 0.8507913947105408, + "learning_rate": 6.100203434089098e-05, + "loss": 0.11434574127197265, + "mean_token_accuracy": 0.9619512343406678, + "num_tokens": 4298066.0, + "step": 3050 + }, + { + "entropy": 0.16490812174975872, + "epoch": 7.452466907340553, + "grad_norm": 0.4558990001678467, + "learning_rate": 5.5995534336748996e-05, + "loss": 0.11507536888122559, + "mean_token_accuracy": 0.9616779097914696, + "num_tokens": 4370032.0, + "step": 3100 + }, + { + "entropy": 0.16860513798892499, + "epoch": 7.57280385078219, + "grad_norm": 0.5815304517745972, + "learning_rate": 5.115625427278985e-05, + "loss": 0.11514325141906738, + "mean_token_accuracy": 0.9618480163812637, + "num_tokens": 4440775.0, + "step": 3150 + }, + { + "entropy": 0.16750167582184075, + "epoch": 7.693140794223827, + "grad_norm": 0.3652438819408417, + "learning_rate": 4.649271111916321e-05, + "loss": 0.11638429641723633, + "mean_token_accuracy": 0.9625930792093277, + "num_tokens": 4512445.0, + "step": 3200 + }, + { + "entropy": 0.15914630327373744, + "epoch": 7.813477737665464, + "grad_norm": 0.33481356501579285, + "learning_rate": 4.2013112554967366e-05, + "loss": 0.1164816951751709, + "mean_token_accuracy": 0.9614231371879578, + "num_tokens": 4582939.0, + "step": 3250 + }, + { + "entropy": 0.16457834955304862, + "epoch": 7.9338146811070995, + "grad_norm": 0.3440433442592621, + "learning_rate": 3.772534252300965e-05, + "loss": 0.11827346801757813, + "mean_token_accuracy": 0.9608591759204864, + "num_tokens": 4651557.0, + "step": 3300 + }, + { + "epoch": 8.0, + "eval_entropy": 0.2139671427916877, + "eval_loss": 0.9454556703567505, + "eval_mean_token_accuracy": 0.8356399822838699, + "eval_num_tokens": 4689712.0, + "eval_runtime": 92.8188, + "eval_samples_per_second": 13.596, + "eval_steps_per_second": 1.702, + "step": 3328 + }, + { + "entropy": 0.15887961709740187, + "epoch": 8.05294825511432, + "grad_norm": 0.27599775791168213, + "learning_rate": 3.363694735433288e-05, + "loss": 0.11239871978759766, + "mean_token_accuracy": 0.9625799821482764, + "num_tokens": 4721916.0, + "step": 3350 + }, + { + "entropy": 0.14713777851313353, + "epoch": 8.173285198555957, + "grad_norm": 0.20361743867397308, + "learning_rate": 2.975512248692604e-05, + "loss": 0.10387100219726562, + "mean_token_accuracy": 0.9654968097805977, + "num_tokens": 4791261.0, + "step": 3400 + }, + { + "entropy": 0.14978370755910875, + "epoch": 8.293622141997593, + "grad_norm": 0.23677319288253784, + "learning_rate": 2.60866998019956e-05, + "loss": 0.1029586124420166, + "mean_token_accuracy": 0.9655927100777626, + "num_tokens": 4860316.0, + "step": 3450 + }, + { + "entropy": 0.15628200586885213, + "epoch": 8.41395908543923, + "grad_norm": 0.26570937037467957, + "learning_rate": 2.2638135600084464e-05, + "loss": 0.10688562393188476, + "mean_token_accuracy": 0.963056892156601, + "num_tokens": 4928346.0, + "step": 3500 + }, + { + "entropy": 0.14125081844627857, + "epoch": 8.534296028880867, + "grad_norm": 0.270812064409256, + "learning_rate": 1.9415499238200032e-05, + "loss": 0.09863008499145508, + "mean_token_accuracy": 0.9660852405428887, + "num_tokens": 5002213.0, + "step": 3550 + }, + { + "entropy": 0.149638747908175, + "epoch": 8.654632972322503, + "grad_norm": 0.3265114426612854, + "learning_rate": 1.6424462447949994e-05, + "loss": 0.10322360992431641, + "mean_token_accuracy": 0.9644484594464302, + "num_tokens": 5074008.0, + "step": 3600 + }, + { + "entropy": 0.14089420825242996, + "epoch": 8.774969915764139, + "grad_norm": 0.4917376637458801, + "learning_rate": 1.3670289353485563e-05, + "loss": 0.10170619964599609, + "mean_token_accuracy": 0.9653348797559738, + "num_tokens": 5145997.0, + "step": 3650 + }, + { + "entropy": 0.14829959750175475, + "epoch": 8.895306859205776, + "grad_norm": 1.1421180963516235, + "learning_rate": 1.1157827206820012e-05, + "loss": 0.10475859642028809, + "mean_token_accuracy": 0.9647275444865226, + "num_tokens": 5215661.0, + "step": 3700 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2012658653002751, + "eval_loss": 1.038285732269287, + "eval_mean_token_accuracy": 0.8358011106147042, + "eval_num_tokens": 5275926.0, + "eval_runtime": 92.6706, + "eval_samples_per_second": 13.618, + "eval_steps_per_second": 1.705, + "step": 3744 + }, + { + "entropy": 0.14548226005651735, + "epoch": 9.014440433212997, + "grad_norm": 0.18666675686836243, + "learning_rate": 8.891497856828061e-06, + "loss": 0.10238224029541015, + "mean_token_accuracy": 0.9649537572957049, + "num_tokens": 5284952.0, + "step": 3750 + }, + { + "entropy": 0.14091712936758996, + "epoch": 9.134777376654633, + "grad_norm": 0.19851620495319366, + "learning_rate": 6.875289966940621e-06, + "loss": 0.09465012550354004, + "mean_token_accuracy": 0.9679439866542816, + "num_tokens": 5357462.0, + "step": 3800 + }, + { + "entropy": 0.13855238482356072, + "epoch": 9.25511432009627, + "grad_norm": 0.24211233854293823, + "learning_rate": 5.11275199523142e-06, + "loss": 0.0947939395904541, + "mean_token_accuracy": 0.9675101634860038, + "num_tokens": 5429842.0, + "step": 3850 + }, + { + "entropy": 0.1402221855521202, + "epoch": 9.375451263537906, + "grad_norm": 0.2897412180900574, + "learning_rate": 3.606985949249944e-06, + "loss": 0.0954015827178955, + "mean_token_accuracy": 0.9671367588639259, + "num_tokens": 5501125.0, + "step": 3900 + }, + { + "entropy": 0.13842488199472427, + "epoch": 9.495788206979542, + "grad_norm": 0.22224950790405273, + "learning_rate": 2.3606419265926856e-06, + "loss": 0.09746825218200683, + "mean_token_accuracy": 0.9666384714841842, + "num_tokens": 5571188.0, + "step": 3950 + }, + { + "entropy": 0.13925437081605196, + "epoch": 9.61612515042118, + "grad_norm": 0.24300234019756317, + "learning_rate": 1.3759134508203263e-06, + "loss": 0.09942876815795898, + "mean_token_accuracy": 0.9659446936845779, + "num_tokens": 5640335.0, + "step": 4000 + }, + { + "entropy": 0.13786451611667871, + "epoch": 9.736462093862816, + "grad_norm": 0.2857815623283386, + "learning_rate": 6.545336109301777e-07, + "loss": 0.0969043254852295, + "mean_token_accuracy": 0.9666687312722206, + "num_tokens": 5711388.0, + "step": 4050 + }, + { + "entropy": 0.1452773343399167, + "epoch": 9.856799037304452, + "grad_norm": 0.21894223988056183, + "learning_rate": 1.9777201117777826e-07, + "loss": 0.10058599472045898, + "mean_token_accuracy": 0.9650132760405541, + "num_tokens": 5779733.0, + "step": 4100 + }, + { + "entropy": 0.1419769449159503, + "epoch": 9.97713598074609, + "grad_norm": 0.3055340647697449, + "learning_rate": 6.432536616100996e-09, + "loss": 0.09695975303649902, + "mean_token_accuracy": 0.9671639212965966, + "num_tokens": 5850073.0, + "step": 4150 + }, + { + "epoch": 10.0, + "eval_entropy": 0.19783954341200333, + "eval_loss": 1.0838706493377686, + "eval_mean_token_accuracy": 0.8355000241647793, + "eval_num_tokens": 5862140.0, + "eval_runtime": 93.1288, + "eval_samples_per_second": 13.551, + "eval_steps_per_second": 1.697, + "step": 4160 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1177399675614822e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/README.md b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/README.md new file mode 100644 index 0000000000000000000000000000000000000000..41e6c854e77830e9ea767c8c35f8c82a65c1ba35 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-4B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-4B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.18.1 \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/adapter_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..bbca7a97b9e49d40f2cdb3ba39ccfb08bb0e1018 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/adapter_config.json @@ -0,0 +1,46 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-4B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.07104296128667408, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.18.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "down_proj", + "up_proj", + "q_proj", + "k_proj", + "v_proj", + "o_proj", + "gate_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/chat_template.jinja b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..a585dec894e63da457d9440ec6aa7caa16d20860 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/tokenizer_config.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/trainer_state.json b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d5527004f2bf2832581965443e5185d408cbb2c3 --- /dev/null +++ b/productivity_original_Swedish/Qwen3.5-4B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-832/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 832, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.621582981646061, + "epoch": 0.12033694344163658, + "grad_norm": 1.7840632200241089, + "learning_rate": 3.557416011989973e-05, + "loss": 1.4308599853515624, + "mean_token_accuracy": 0.6972626076638698, + "num_tokens": 74350.0, + "step": 50 + }, + { + "entropy": 0.7890771624445915, + "epoch": 0.24067388688327315, + "grad_norm": 1.1691383123397827, + "learning_rate": 7.187432350755252e-05, + "loss": 0.68025390625, + "mean_token_accuracy": 0.8160018160939216, + "num_tokens": 147142.0, + "step": 100 + }, + { + "entropy": 0.7458965721726417, + "epoch": 0.36101083032490977, + "grad_norm": 0.9552986025810242, + "learning_rate": 0.0001081744868952053, + "loss": 0.6382268524169922, + "mean_token_accuracy": 0.8264451423287391, + "num_tokens": 218282.0, + "step": 150 + }, + { + "entropy": 0.735072555243969, + "epoch": 0.4813477737665463, + "grad_norm": 1.0810377597808838, + "learning_rate": 0.0001444746502828581, + "loss": 0.6294943237304688, + "mean_token_accuracy": 0.8261765521764756, + "num_tokens": 288979.0, + "step": 200 + }, + { + "entropy": 0.7114953261613846, + "epoch": 0.601684717208183, + "grad_norm": 0.9742010831832886, + "learning_rate": 0.00018077481367051088, + "loss": 0.602623176574707, + "mean_token_accuracy": 0.83393067330122, + "num_tokens": 358762.0, + "step": 250 + }, + { + "entropy": 0.7069582852721215, + "epoch": 0.7220216606498195, + "grad_norm": 0.7885417342185974, + "learning_rate": 0.00021707497705816367, + "loss": 0.6058937072753906, + "mean_token_accuracy": 0.8321960058808326, + "num_tokens": 428124.0, + "step": 300 + }, + { + "entropy": 0.7101716002821923, + "epoch": 0.8423586040914561, + "grad_norm": 0.8627666234970093, + "learning_rate": 0.00025337514044581645, + "loss": 0.6131188583374023, + "mean_token_accuracy": 0.8330579137802124, + "num_tokens": 497622.0, + "step": 350 + }, + { + "entropy": 0.6990390625596047, + "epoch": 0.9626955475330926, + "grad_norm": 1.1322276592254639, + "learning_rate": 0.00028967530383346927, + "loss": 0.6065136337280274, + "mean_token_accuracy": 0.8326930233836174, + "num_tokens": 565234.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.6208525450923775, + "eval_loss": 0.6319083571434021, + "eval_mean_token_accuracy": 0.8220708449430103, + "eval_num_tokens": 586214.0, + "eval_runtime": 93.9256, + "eval_samples_per_second": 13.436, + "eval_steps_per_second": 1.682, + "step": 416 + }, + { + "entropy": 0.6688173970188758, + "epoch": 1.0818291215403129, + "grad_norm": 1.2222388982772827, + "learning_rate": 0.0003019594698437709, + "loss": 0.5917711639404297, + "mean_token_accuracy": 0.8361045230518688, + "num_tokens": 634164.0, + "step": 450 + }, + { + "entropy": 0.6467562046647072, + "epoch": 1.2021660649819494, + "grad_norm": 1.0695290565490723, + "learning_rate": 0.00030165127549903, + "loss": 0.5744387817382812, + "mean_token_accuracy": 0.8404921460151672, + "num_tokens": 703202.0, + "step": 500 + }, + { + "entropy": 0.6470985013246536, + "epoch": 1.322503008423586, + "grad_norm": 1.0010168552398682, + "learning_rate": 0.0003010779552523376, + "loss": 0.574291648864746, + "mean_token_accuracy": 0.8386730599403381, + "num_tokens": 775825.0, + "step": 550 + }, + { + "entropy": 0.6490107670426368, + "epoch": 1.4428399518652226, + "grad_norm": 0.9982119202613831, + "learning_rate": 0.0003002405181280448, + "loss": 0.5834595870971679, + "mean_token_accuracy": 0.8363789242506027, + "num_tokens": 846212.0, + "step": 600 + }, + { + "entropy": 0.6380249202251435, + "epoch": 1.5631768953068592, + "grad_norm": 1.0275771617889404, + "learning_rate": 0.00029914043798732445, + "loss": 0.5688864517211915, + "mean_token_accuracy": 0.8405828747153282, + "num_tokens": 917553.0, + "step": 650 + }, + { + "entropy": 0.6351437552273274, + "epoch": 1.6835138387484958, + "grad_norm": 0.9777424931526184, + "learning_rate": 0.0002977796509342244, + "loss": 0.5608405303955079, + "mean_token_accuracy": 0.8426707127690315, + "num_tokens": 987574.0, + "step": 700 + }, + { + "entropy": 0.6300888039171696, + "epoch": 1.8038507821901324, + "grad_norm": 1.1589821577072144, + "learning_rate": 0.00029616055190818973, + "loss": 0.5640585708618164, + "mean_token_accuracy": 0.8411854481697083, + "num_tokens": 1058374.0, + "step": 750 + }, + { + "entropy": 0.6372530253231525, + "epoch": 1.924187725631769, + "grad_norm": 1.3437201976776123, + "learning_rate": 0.00029428599046905007, + "loss": 0.5722800445556641, + "mean_token_accuracy": 0.8391004773974419, + "num_tokens": 1128418.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.5832181644590595, + "eval_loss": 0.5960000157356262, + "eval_mean_token_accuracy": 0.8264509171624727, + "eval_num_tokens": 1172428.0, + "eval_runtime": 93.0484, + "eval_samples_per_second": 13.563, + "eval_steps_per_second": 1.698, + "step": 832 + } + ], + "logging_steps": 50, + "max_steps": 4160, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.226422000754688e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}