diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..91eb2858d06e06baa46041257244c956622b8b7c --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/hlci69my) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..42d72ea93d587a366dd31015463d240c6c489910 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 +tags: +- generated_from_trainer +- trl +- sft +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/qqnxtz0d) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c7ff2f7a9b8ad0b165fe85ff8cd688d64d51e60b --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1167/trainer_state.json @@ -0,0 +1,297 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1167, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.936105194091797, + "epoch": 0.1287001287001287, + "grad_norm": 2.5512146949768066, + "learning_rate": 7.410977855063067e-06, + "loss": 2.765989074707031, + "mean_token_accuracy": 0.5075639402866363, + "num_tokens": 69648.0, + "step": 50 + }, + { + "entropy": 1.4886422049999237, + "epoch": 0.2574002574002574, + "grad_norm": 2.2254061698913574, + "learning_rate": 1.4973200156147831e-05, + "loss": 1.336165313720703, + "mean_token_accuracy": 0.6740516322851181, + "num_tokens": 136672.0, + "step": 100 + }, + { + "entropy": 1.1471627122163772, + "epoch": 0.3861003861003861, + "grad_norm": 2.2073140144348145, + "learning_rate": 2.2535422457232594e-05, + "loss": 1.0278042602539061, + "mean_token_accuracy": 0.7280837368965148, + "num_tokens": 205085.0, + "step": 150 + }, + { + "entropy": 1.0486015713214873, + "epoch": 0.5148005148005148, + "grad_norm": 1.650594711303711, + "learning_rate": 3.0097644758317354e-05, + "loss": 0.933094253540039, + "mean_token_accuracy": 0.7478678917884827, + "num_tokens": 270093.0, + "step": 200 + }, + { + "entropy": 0.9942852139472962, + "epoch": 0.6435006435006435, + "grad_norm": 1.7949399948120117, + "learning_rate": 3.765986705940212e-05, + "loss": 0.8747442626953125, + "mean_token_accuracy": 0.7603094267845154, + "num_tokens": 338177.0, + "step": 250 + }, + { + "entropy": 0.9491432571411133, + "epoch": 0.7722007722007722, + "grad_norm": 1.33268404006958, + "learning_rate": 4.522208936048689e-05, + "loss": 0.8404794311523438, + "mean_token_accuracy": 0.7671167278289794, + "num_tokens": 405099.0, + "step": 300 + }, + { + "entropy": 0.9246599394083023, + "epoch": 0.9009009009009009, + "grad_norm": 1.3137660026550293, + "learning_rate": 5.278431166157165e-05, + "loss": 0.8163955688476563, + "mean_token_accuracy": 0.7712420308589936, + "num_tokens": 474117.0, + "step": 350 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7913847528782088, + "eval_loss": 0.7582769393920898, + "eval_mean_token_accuracy": 0.7787086770092089, + "eval_num_tokens": 527146.0, + "eval_runtime": 80.7434, + "eval_samples_per_second": 19.135, + "eval_steps_per_second": 2.403, + "step": 389 + }, + { + "entropy": 0.9051812700550965, + "epoch": 1.0283140283140284, + "grad_norm": 1.3198363780975342, + "learning_rate": 5.883290514810127e-05, + "loss": 0.7887770843505859, + "mean_token_accuracy": 0.7787209345836832, + "num_tokens": 541730.0, + "step": 400 + }, + { + "entropy": 0.8626887363195419, + "epoch": 1.157014157014157, + "grad_norm": 1.1772489547729492, + "learning_rate": 5.8791462758850675e-05, + "loss": 0.7505465698242187, + "mean_token_accuracy": 0.7887020426988601, + "num_tokens": 610116.0, + "step": 450 + }, + { + "entropy": 0.8474252647161484, + "epoch": 1.2857142857142856, + "grad_norm": 1.0174403190612793, + "learning_rate": 5.8690897984332274e-05, + "loss": 0.7397904968261719, + "mean_token_accuracy": 0.7890202105045319, + "num_tokens": 677990.0, + "step": 500 + }, + { + "entropy": 0.843030309677124, + "epoch": 1.4144144144144144, + "grad_norm": 1.1678341627120972, + "learning_rate": 5.853141323295946e-05, + "loss": 0.7369876098632813, + "mean_token_accuracy": 0.790421934723854, + "num_tokens": 745456.0, + "step": 550 + }, + { + "entropy": 0.8115463078022003, + "epoch": 1.5431145431145432, + "grad_norm": 0.8854549527168274, + "learning_rate": 5.831332950237421e-05, + "loss": 0.7081725311279297, + "mean_token_accuracy": 0.7952603316307068, + "num_tokens": 813775.0, + "step": 600 + }, + { + "entropy": 0.8311941117048264, + "epoch": 1.6718146718146718, + "grad_norm": 1.171217441558838, + "learning_rate": 5.803708573336975e-05, + "loss": 0.7232355499267578, + "mean_token_accuracy": 0.7947697073221207, + "num_tokens": 879616.0, + "step": 650 + }, + { + "entropy": 0.8120440804958343, + "epoch": 1.8005148005148004, + "grad_norm": 1.0333378314971924, + "learning_rate": 5.770323792642697e-05, + "loss": 0.6972620391845703, + "mean_token_accuracy": 0.7987716430425644, + "num_tokens": 947922.0, + "step": 700 + }, + { + "entropy": 0.8045441079139709, + "epoch": 1.9292149292149292, + "grad_norm": 1.01642906665802, + "learning_rate": 5.7312458022642996e-05, + "loss": 0.6930023193359375, + "mean_token_accuracy": 0.8009403163194656, + "num_tokens": 1014787.0, + "step": 750 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6984818912658495, + "eval_loss": 0.6775499582290649, + "eval_mean_token_accuracy": 0.8020080097557343, + "eval_num_tokens": 1054292.0, + "eval_runtime": 78.3908, + "eval_samples_per_second": 19.709, + "eval_steps_per_second": 2.475, + "step": 778 + }, + { + "entropy": 0.7779398566544659, + "epoch": 2.056628056628057, + "grad_norm": 1.4157040119171143, + "learning_rate": 5.686553255130396e-05, + "loss": 0.6584475708007812, + "mean_token_accuracy": 0.8087991054611977, + "num_tokens": 1083073.0, + "step": 800 + }, + { + "entropy": 0.745064823627472, + "epoch": 2.1853281853281854, + "grad_norm": 0.9885228872299194, + "learning_rate": 5.636336104682433e-05, + "loss": 0.618268051147461, + "mean_token_accuracy": 0.8165298491716385, + "num_tokens": 1149040.0, + "step": 850 + }, + { + "entropy": 0.7321632200479508, + "epoch": 2.314028314028314, + "grad_norm": 1.0271121263504028, + "learning_rate": 5.580695423823882e-05, + "loss": 0.6195047760009765, + "mean_token_accuracy": 0.8162743961811065, + "num_tokens": 1218025.0, + "step": 900 + }, + { + "entropy": 0.7240813142061233, + "epoch": 2.4427284427284426, + "grad_norm": 0.8744781613349915, + "learning_rate": 5.519743201489109e-05, + "loss": 0.6093913650512696, + "mean_token_accuracy": 0.8179467821121216, + "num_tokens": 1287270.0, + "step": 950 + }, + { + "entropy": 0.7273010468482971, + "epoch": 2.571428571428571, + "grad_norm": 1.0161406993865967, + "learning_rate": 5.453602117241361e-05, + "loss": 0.6118500518798828, + "mean_token_accuracy": 0.8183792388439178, + "num_tokens": 1355064.0, + "step": 1000 + }, + { + "entropy": 0.7150553166866302, + "epoch": 2.7001287001287, + "grad_norm": 0.9795015454292297, + "learning_rate": 5.382405294353544e-05, + "loss": 0.6005525970458985, + "mean_token_accuracy": 0.8210368871688842, + "num_tokens": 1421622.0, + "step": 1050 + }, + { + "entropy": 0.7134475702047348, + "epoch": 2.828828828828829, + "grad_norm": 0.9153344035148621, + "learning_rate": 5.306296031868776e-05, + "loss": 0.6033024978637695, + "mean_token_accuracy": 0.8216253930330276, + "num_tokens": 1488775.0, + "step": 1100 + }, + { + "entropy": 0.7142637860774994, + "epoch": 2.9575289575289574, + "grad_norm": 0.8638553619384766, + "learning_rate": 5.2254275161799905e-05, + "loss": 0.6056064605712891, + "mean_token_accuracy": 0.8204184120893478, + "num_tokens": 1559605.0, + "step": 1150 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6126322589584232, + "eval_loss": 0.676252007484436, + "eval_mean_token_accuracy": 0.8002185584958067, + "eval_num_tokens": 1581438.0, + "eval_runtime": 78.2302, + "eval_samples_per_second": 19.749, + "eval_steps_per_second": 2.48, + "step": 1167 + } + ], + "logging_steps": 50, + "max_steps": 3890, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.664243134168064e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f298f9d53609598cf56ba3b881c57739258a0df9 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.06250292706368335, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "gate_proj", + "up_proj", + "v_proj", + "down_proj", + "o_proj", + "k_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/productivity_original_Estonian/Qwen3.5-2B-Base_productivity_splits_original_features_train_productivity_splits_original_features_test2/checkpoint-1556/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..72440419b4f8474af058e8fee6c0d65355bbdc48 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/hey06srd) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3bb61133ff99ce0f07c162deee3861085532a1 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1224/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.580693020417024e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..028fe358b527995f192671457655f41a2333d92e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-1632/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.779536115351488e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..979284518bb21fd23472811439cdf248b0d71849 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2040/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0964287730493312e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e484eaf1e7deff0c003615d730fc6fd5d4de9fbf --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2448/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + }, + { + "entropy": 0.4460540203133015, + "epoch": 5.024539877300613, + "grad_norm": 0.6123270988464355, + "learning_rate": 0.00025555371804224084, + "loss": 0.40383037567138674, + "mean_token_accuracy": 0.8667331608858976, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.34027856796979905, + "epoch": 5.147239263803681, + "grad_norm": 0.6245328187942505, + "learning_rate": 0.00024627825624807994, + "loss": 0.29572994232177735, + "mean_token_accuracy": 0.8982829064130783, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.3554780498147011, + "epoch": 5.269938650306749, + "grad_norm": 0.5149081349372864, + "learning_rate": 0.0002369531774958907, + "loss": 0.3053621292114258, + "mean_token_accuracy": 0.8955890852212905, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3499211978912353, + "epoch": 5.392638036809816, + "grad_norm": 0.5816537737846375, + "learning_rate": 0.0002275955434223227, + "loss": 0.3001568603515625, + "mean_token_accuracy": 0.8958922523260117, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3544818899035454, + "epoch": 5.515337423312883, + "grad_norm": 0.6629130244255066, + "learning_rate": 0.0002182224752288907, + "loss": 0.3112427520751953, + "mean_token_accuracy": 0.8927132803201675, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.35083883732557297, + "epoch": 5.638036809815951, + "grad_norm": 0.6243860125541687, + "learning_rate": 0.00020885112235615794, + "loss": 0.3076825332641602, + "mean_token_accuracy": 0.8949526751041412, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.3593694043159485, + "epoch": 5.7607361963190185, + "grad_norm": 0.6104142069816589, + "learning_rate": 0.00019949863110624958, + "loss": 0.3147741317749023, + "mean_token_accuracy": 0.8915749222040177, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.35077930554747583, + "epoch": 5.883435582822086, + "grad_norm": 0.5526567697525024, + "learning_rate": 0.00019018211327110994, + "loss": 0.3106169509887695, + "mean_token_accuracy": 0.8925997793674469, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.43280206646238056, + "eval_loss": 0.6834793090820312, + "eval_mean_token_accuracy": 0.8267195994513376, + "eval_num_tokens": 6262626.0, + "eval_runtime": 56.0697, + "eval_samples_per_second": 24.898, + "eval_steps_per_second": 3.121, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3153170392634048e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..a495d44e55fdc46a52384c2272930aa2c46b6714 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-2856/trainer_state.json @@ -0,0 +1,681 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + }, + { + "entropy": 0.4460540203133015, + "epoch": 5.024539877300613, + "grad_norm": 0.6123270988464355, + "learning_rate": 0.00025555371804224084, + "loss": 0.40383037567138674, + "mean_token_accuracy": 0.8667331608858976, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.34027856796979905, + "epoch": 5.147239263803681, + "grad_norm": 0.6245328187942505, + "learning_rate": 0.00024627825624807994, + "loss": 0.29572994232177735, + "mean_token_accuracy": 0.8982829064130783, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.3554780498147011, + "epoch": 5.269938650306749, + "grad_norm": 0.5149081349372864, + "learning_rate": 0.0002369531774958907, + "loss": 0.3053621292114258, + "mean_token_accuracy": 0.8955890852212905, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3499211978912353, + "epoch": 5.392638036809816, + "grad_norm": 0.5816537737846375, + "learning_rate": 0.0002275955434223227, + "loss": 0.3001568603515625, + "mean_token_accuracy": 0.8958922523260117, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3544818899035454, + "epoch": 5.515337423312883, + "grad_norm": 0.6629130244255066, + "learning_rate": 0.0002182224752288907, + "loss": 0.3112427520751953, + "mean_token_accuracy": 0.8927132803201675, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.35083883732557297, + "epoch": 5.638036809815951, + "grad_norm": 0.6243860125541687, + "learning_rate": 0.00020885112235615794, + "loss": 0.3076825332641602, + "mean_token_accuracy": 0.8949526751041412, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.3593694043159485, + "epoch": 5.7607361963190185, + "grad_norm": 0.6104142069816589, + "learning_rate": 0.00019949863110624958, + "loss": 0.3147741317749023, + "mean_token_accuracy": 0.8915749222040177, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.35077930554747583, + "epoch": 5.883435582822086, + "grad_norm": 0.5526567697525024, + "learning_rate": 0.00019018211327110994, + "loss": 0.3106169509887695, + "mean_token_accuracy": 0.8925997793674469, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.43280206646238056, + "eval_loss": 0.6834793090820312, + "eval_mean_token_accuracy": 0.8267195994513376, + "eval_num_tokens": 6262626.0, + "eval_runtime": 56.0697, + "eval_samples_per_second": 24.898, + "eval_steps_per_second": 3.121, + "step": 2448 + }, + { + "entropy": 0.35777792816210274, + "epoch": 6.004907975460123, + "grad_norm": 0.49246102571487427, + "learning_rate": 0.00018091861482390074, + "loss": 0.3066068077087402, + "mean_token_accuracy": 0.8934184981114937, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.23881351679563523, + "epoch": 6.12760736196319, + "grad_norm": 0.5304619073867798, + "learning_rate": 0.0001717250847308245, + "loss": 0.189466609954834, + "mean_token_accuracy": 0.9343969762325287, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.23922096833586692, + "epoch": 6.250306748466258, + "grad_norm": 0.5296095609664917, + "learning_rate": 0.0001626183439404375, + "loss": 0.1947933006286621, + "mean_token_accuracy": 0.9307195580005646, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.24290484443306923, + "epoch": 6.373006134969325, + "grad_norm": 0.69776451587677, + "learning_rate": 0.00015361505460718933, + "loss": 0.19909774780273437, + "mean_token_accuracy": 0.9303810286521912, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.24054641783237457, + "epoch": 6.495705521472392, + "grad_norm": 0.5774877071380615, + "learning_rate": 0.00014473168960550125, + "loss": 0.1948543930053711, + "mean_token_accuracy": 0.9311885589361191, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.24378441721200944, + "epoch": 6.61840490797546, + "grad_norm": 0.6684483289718628, + "learning_rate": 0.0001359845023901601, + "loss": 0.20006532669067384, + "mean_token_accuracy": 0.9293528574705124, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.24649206757545472, + "epoch": 6.741104294478528, + "grad_norm": 0.5416299104690552, + "learning_rate": 0.0001273894972581737, + "loss": 0.20033370971679687, + "mean_token_accuracy": 0.9294078814983368, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.245157231092453, + "epoch": 6.863803680981595, + "grad_norm": 0.6425517201423645, + "learning_rate": 0.00011896240006649818, + "loss": 0.19885097503662108, + "mean_token_accuracy": 0.929965842962265, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.23941567167639732, + "epoch": 6.986503067484662, + "grad_norm": 0.5717498660087585, + "learning_rate": 0.00011071862945921291, + "loss": 0.19223129272460937, + "mean_token_accuracy": 0.9328319728374481, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.34101752851690564, + "eval_loss": 0.7636436223983765, + "eval_mean_token_accuracy": 0.8270581153460911, + "eval_num_tokens": 7306397.0, + "eval_runtime": 57.0887, + "eval_samples_per_second": 24.453, + "eval_steps_per_second": 3.065, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5332087363966208e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6a29b07e0c8d9983f47414d3d4369f15a063955e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3264/trainer_state.json @@ -0,0 +1,772 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + }, + { + "entropy": 0.4460540203133015, + "epoch": 5.024539877300613, + "grad_norm": 0.6123270988464355, + "learning_rate": 0.00025555371804224084, + "loss": 0.40383037567138674, + "mean_token_accuracy": 0.8667331608858976, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.34027856796979905, + "epoch": 5.147239263803681, + "grad_norm": 0.6245328187942505, + "learning_rate": 0.00024627825624807994, + "loss": 0.29572994232177735, + "mean_token_accuracy": 0.8982829064130783, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.3554780498147011, + "epoch": 5.269938650306749, + "grad_norm": 0.5149081349372864, + "learning_rate": 0.0002369531774958907, + "loss": 0.3053621292114258, + "mean_token_accuracy": 0.8955890852212905, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3499211978912353, + "epoch": 5.392638036809816, + "grad_norm": 0.5816537737846375, + "learning_rate": 0.0002275955434223227, + "loss": 0.3001568603515625, + "mean_token_accuracy": 0.8958922523260117, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3544818899035454, + "epoch": 5.515337423312883, + "grad_norm": 0.6629130244255066, + "learning_rate": 0.0002182224752288907, + "loss": 0.3112427520751953, + "mean_token_accuracy": 0.8927132803201675, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.35083883732557297, + "epoch": 5.638036809815951, + "grad_norm": 0.6243860125541687, + "learning_rate": 0.00020885112235615794, + "loss": 0.3076825332641602, + "mean_token_accuracy": 0.8949526751041412, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.3593694043159485, + "epoch": 5.7607361963190185, + "grad_norm": 0.6104142069816589, + "learning_rate": 0.00019949863110624958, + "loss": 0.3147741317749023, + "mean_token_accuracy": 0.8915749222040177, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.35077930554747583, + "epoch": 5.883435582822086, + "grad_norm": 0.5526567697525024, + "learning_rate": 0.00019018211327110994, + "loss": 0.3106169509887695, + "mean_token_accuracy": 0.8925997793674469, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.43280206646238056, + "eval_loss": 0.6834793090820312, + "eval_mean_token_accuracy": 0.8267195994513376, + "eval_num_tokens": 6262626.0, + "eval_runtime": 56.0697, + "eval_samples_per_second": 24.898, + "eval_steps_per_second": 3.121, + "step": 2448 + }, + { + "entropy": 0.35777792816210274, + "epoch": 6.004907975460123, + "grad_norm": 0.49246102571487427, + "learning_rate": 0.00018091861482390074, + "loss": 0.3066068077087402, + "mean_token_accuracy": 0.8934184981114937, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.23881351679563523, + "epoch": 6.12760736196319, + "grad_norm": 0.5304619073867798, + "learning_rate": 0.0001717250847308245, + "loss": 0.189466609954834, + "mean_token_accuracy": 0.9343969762325287, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.23922096833586692, + "epoch": 6.250306748466258, + "grad_norm": 0.5296095609664917, + "learning_rate": 0.0001626183439404375, + "loss": 0.1947933006286621, + "mean_token_accuracy": 0.9307195580005646, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.24290484443306923, + "epoch": 6.373006134969325, + "grad_norm": 0.69776451587677, + "learning_rate": 0.00015361505460718933, + "loss": 0.19909774780273437, + "mean_token_accuracy": 0.9303810286521912, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.24054641783237457, + "epoch": 6.495705521472392, + "grad_norm": 0.5774877071380615, + "learning_rate": 0.00014473168960550125, + "loss": 0.1948543930053711, + "mean_token_accuracy": 0.9311885589361191, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.24378441721200944, + "epoch": 6.61840490797546, + "grad_norm": 0.6684483289718628, + "learning_rate": 0.0001359845023901601, + "loss": 0.20006532669067384, + "mean_token_accuracy": 0.9293528574705124, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.24649206757545472, + "epoch": 6.741104294478528, + "grad_norm": 0.5416299104690552, + "learning_rate": 0.0001273894972581737, + "loss": 0.20033370971679687, + "mean_token_accuracy": 0.9294078814983368, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.245157231092453, + "epoch": 6.863803680981595, + "grad_norm": 0.6425517201423645, + "learning_rate": 0.00011896240006649818, + "loss": 0.19885097503662108, + "mean_token_accuracy": 0.929965842962265, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.23941567167639732, + "epoch": 6.986503067484662, + "grad_norm": 0.5717498660087585, + "learning_rate": 0.00011071862945921291, + "loss": 0.19223129272460937, + "mean_token_accuracy": 0.9328319728374481, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.34101752851690564, + "eval_loss": 0.7636436223983765, + "eval_mean_token_accuracy": 0.8270581153460911, + "eval_num_tokens": 7306397.0, + "eval_runtime": 57.0887, + "eval_samples_per_second": 24.453, + "eval_steps_per_second": 3.065, + "step": 2856 + }, + { + "entropy": 0.17031941931657116, + "epoch": 7.1079754601227, + "grad_norm": 0.4665500819683075, + "learning_rate": 0.0001026732686567881, + "loss": 0.1262428092956543, + "mean_token_accuracy": 0.9578677494116504, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.1474052606523037, + "epoch": 7.230674846625767, + "grad_norm": 0.45524656772613525, + "learning_rate": 9.484103785906042e-05, + "loss": 0.10753781318664551, + "mean_token_accuracy": 0.9643174183368682, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.14507727995514869, + "epoch": 7.353374233128834, + "grad_norm": 0.5161148309707642, + "learning_rate": 8.723626731240899e-05, + "loss": 0.10872675895690918, + "mean_token_accuracy": 0.9626953148841858, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.14192182645201684, + "epoch": 7.476073619631902, + "grad_norm": 0.5691169500350952, + "learning_rate": 7.987287109041075e-05, + "loss": 0.10497721672058105, + "mean_token_accuracy": 0.9646239751577377, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.14175789937376976, + "epoch": 7.598773006134969, + "grad_norm": 0.5878594517707825, + "learning_rate": 7.276432163594605e-05, + "loss": 0.10868305206298828, + "mean_token_accuracy": 0.9635398960113526, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.14344056263566018, + "epoch": 7.721472392638037, + "grad_norm": 0.5571667551994324, + "learning_rate": 6.592362511133478e-05, + "loss": 0.10832572937011718, + "mean_token_accuracy": 0.9632837051153182, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.14393038906157016, + "epoch": 7.844171779141105, + "grad_norm": 0.5589098334312439, + "learning_rate": 5.93632976016024e-05, + "loss": 0.10971030235290527, + "mean_token_accuracy": 0.9629397732019425, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.14122579149901868, + "epoch": 7.9668711656441715, + "grad_norm": 0.6026595830917358, + "learning_rate": 5.309534221441647e-05, + "loss": 0.10822047233581543, + "mean_token_accuracy": 0.9636554193496704, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28723810638700215, + "eval_loss": 0.864037811756134, + "eval_mean_token_accuracy": 0.8260653465134757, + "eval_num_tokens": 8350168.0, + "eval_runtime": 56.6604, + "eval_samples_per_second": 24.638, + "eval_steps_per_second": 3.089, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7506329083762688e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..864dc5134a3a0b4339120fe52c26a3fa596f6017 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-3672/trainer_state.json @@ -0,0 +1,863 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + }, + { + "entropy": 0.4460540203133015, + "epoch": 5.024539877300613, + "grad_norm": 0.6123270988464355, + "learning_rate": 0.00025555371804224084, + "loss": 0.40383037567138674, + "mean_token_accuracy": 0.8667331608858976, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.34027856796979905, + "epoch": 5.147239263803681, + "grad_norm": 0.6245328187942505, + "learning_rate": 0.00024627825624807994, + "loss": 0.29572994232177735, + "mean_token_accuracy": 0.8982829064130783, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.3554780498147011, + "epoch": 5.269938650306749, + "grad_norm": 0.5149081349372864, + "learning_rate": 0.0002369531774958907, + "loss": 0.3053621292114258, + "mean_token_accuracy": 0.8955890852212905, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3499211978912353, + "epoch": 5.392638036809816, + "grad_norm": 0.5816537737846375, + "learning_rate": 0.0002275955434223227, + "loss": 0.3001568603515625, + "mean_token_accuracy": 0.8958922523260117, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3544818899035454, + "epoch": 5.515337423312883, + "grad_norm": 0.6629130244255066, + "learning_rate": 0.0002182224752288907, + "loss": 0.3112427520751953, + "mean_token_accuracy": 0.8927132803201675, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.35083883732557297, + "epoch": 5.638036809815951, + "grad_norm": 0.6243860125541687, + "learning_rate": 0.00020885112235615794, + "loss": 0.3076825332641602, + "mean_token_accuracy": 0.8949526751041412, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.3593694043159485, + "epoch": 5.7607361963190185, + "grad_norm": 0.6104142069816589, + "learning_rate": 0.00019949863110624958, + "loss": 0.3147741317749023, + "mean_token_accuracy": 0.8915749222040177, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.35077930554747583, + "epoch": 5.883435582822086, + "grad_norm": 0.5526567697525024, + "learning_rate": 0.00019018211327110994, + "loss": 0.3106169509887695, + "mean_token_accuracy": 0.8925997793674469, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.43280206646238056, + "eval_loss": 0.6834793090820312, + "eval_mean_token_accuracy": 0.8267195994513376, + "eval_num_tokens": 6262626.0, + "eval_runtime": 56.0697, + "eval_samples_per_second": 24.898, + "eval_steps_per_second": 3.121, + "step": 2448 + }, + { + "entropy": 0.35777792816210274, + "epoch": 6.004907975460123, + "grad_norm": 0.49246102571487427, + "learning_rate": 0.00018091861482390074, + "loss": 0.3066068077087402, + "mean_token_accuracy": 0.8934184981114937, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.23881351679563523, + "epoch": 6.12760736196319, + "grad_norm": 0.5304619073867798, + "learning_rate": 0.0001717250847308245, + "loss": 0.189466609954834, + "mean_token_accuracy": 0.9343969762325287, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.23922096833586692, + "epoch": 6.250306748466258, + "grad_norm": 0.5296095609664917, + "learning_rate": 0.0001626183439404375, + "loss": 0.1947933006286621, + "mean_token_accuracy": 0.9307195580005646, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.24290484443306923, + "epoch": 6.373006134969325, + "grad_norm": 0.69776451587677, + "learning_rate": 0.00015361505460718933, + "loss": 0.19909774780273437, + "mean_token_accuracy": 0.9303810286521912, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.24054641783237457, + "epoch": 6.495705521472392, + "grad_norm": 0.5774877071380615, + "learning_rate": 0.00014473168960550125, + "loss": 0.1948543930053711, + "mean_token_accuracy": 0.9311885589361191, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.24378441721200944, + "epoch": 6.61840490797546, + "grad_norm": 0.6684483289718628, + "learning_rate": 0.0001359845023901601, + "loss": 0.20006532669067384, + "mean_token_accuracy": 0.9293528574705124, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.24649206757545472, + "epoch": 6.741104294478528, + "grad_norm": 0.5416299104690552, + "learning_rate": 0.0001273894972581737, + "loss": 0.20033370971679687, + "mean_token_accuracy": 0.9294078814983368, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.245157231092453, + "epoch": 6.863803680981595, + "grad_norm": 0.6425517201423645, + "learning_rate": 0.00011896240006649818, + "loss": 0.19885097503662108, + "mean_token_accuracy": 0.929965842962265, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.23941567167639732, + "epoch": 6.986503067484662, + "grad_norm": 0.5717498660087585, + "learning_rate": 0.00011071862945921291, + "loss": 0.19223129272460937, + "mean_token_accuracy": 0.9328319728374481, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.34101752851690564, + "eval_loss": 0.7636436223983765, + "eval_mean_token_accuracy": 0.8270581153460911, + "eval_num_tokens": 7306397.0, + "eval_runtime": 57.0887, + "eval_samples_per_second": 24.453, + "eval_steps_per_second": 3.065, + "step": 2856 + }, + { + "entropy": 0.17031941931657116, + "epoch": 7.1079754601227, + "grad_norm": 0.4665500819683075, + "learning_rate": 0.0001026732686567881, + "loss": 0.1262428092956543, + "mean_token_accuracy": 0.9578677494116504, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.1474052606523037, + "epoch": 7.230674846625767, + "grad_norm": 0.45524656772613525, + "learning_rate": 9.484103785906042e-05, + "loss": 0.10753781318664551, + "mean_token_accuracy": 0.9643174183368682, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.14507727995514869, + "epoch": 7.353374233128834, + "grad_norm": 0.5161148309707642, + "learning_rate": 8.723626731240899e-05, + "loss": 0.10872675895690918, + "mean_token_accuracy": 0.9626953148841858, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.14192182645201684, + "epoch": 7.476073619631902, + "grad_norm": 0.5691169500350952, + "learning_rate": 7.987287109041075e-05, + "loss": 0.10497721672058105, + "mean_token_accuracy": 0.9646239751577377, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.14175789937376976, + "epoch": 7.598773006134969, + "grad_norm": 0.5878594517707825, + "learning_rate": 7.276432163594605e-05, + "loss": 0.10868305206298828, + "mean_token_accuracy": 0.9635398960113526, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.14344056263566018, + "epoch": 7.721472392638037, + "grad_norm": 0.5571667551994324, + "learning_rate": 6.592362511133478e-05, + "loss": 0.10832572937011718, + "mean_token_accuracy": 0.9632837051153182, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.14393038906157016, + "epoch": 7.844171779141105, + "grad_norm": 0.5589098334312439, + "learning_rate": 5.93632976016024e-05, + "loss": 0.10971030235290527, + "mean_token_accuracy": 0.9629397732019425, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.14122579149901868, + "epoch": 7.9668711656441715, + "grad_norm": 0.6026595830917358, + "learning_rate": 5.309534221441647e-05, + "loss": 0.10822047233581543, + "mean_token_accuracy": 0.9636554193496704, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28723810638700215, + "eval_loss": 0.864037811756134, + "eval_mean_token_accuracy": 0.8260653465134757, + "eval_num_tokens": 8350168.0, + "eval_runtime": 56.6604, + "eval_samples_per_second": 24.638, + "eval_steps_per_second": 3.089, + "step": 3264 + }, + { + "entropy": 0.118123043591928, + "epoch": 8.088343558282208, + "grad_norm": 0.35258734226226807, + "learning_rate": 4.713122711859268e-05, + "loss": 0.08152069091796875, + "mean_token_accuracy": 0.9731789550395927, + "num_tokens": 8438737.0, + "step": 3300 + }, + { + "entropy": 0.10090006463229656, + "epoch": 8.211042944785277, + "grad_norm": 0.3480742573738098, + "learning_rate": 4.148186456135221e-05, + "loss": 0.06941273212432861, + "mean_token_accuracy": 0.9767383986711502, + "num_tokens": 8565559.0, + "step": 3350 + }, + { + "entropy": 0.10347712013870478, + "epoch": 8.333742331288343, + "grad_norm": 0.3159559965133667, + "learning_rate": 3.6157590902720824e-05, + "loss": 0.07076391220092773, + "mean_token_accuracy": 0.9752408146858216, + "num_tokens": 8690700.0, + "step": 3400 + }, + { + "entropy": 0.09924204669892787, + "epoch": 8.45644171779141, + "grad_norm": 0.2995770573616028, + "learning_rate": 3.1168147703600954e-05, + "loss": 0.07081368923187256, + "mean_token_accuracy": 0.9765943318605423, + "num_tokens": 8817258.0, + "step": 3450 + }, + { + "entropy": 0.09427825551480055, + "epoch": 8.579141104294479, + "grad_norm": 0.2867041826248169, + "learning_rate": 2.6522663902118405e-05, + "loss": 0.06653664588928222, + "mean_token_accuracy": 0.9782084548473358, + "num_tokens": 8954362.0, + "step": 3500 + }, + { + "entropy": 0.09728625949472189, + "epoch": 8.701840490797546, + "grad_norm": 0.24146562814712524, + "learning_rate": 2.2229639110854164e-05, + "loss": 0.06928532123565674, + "mean_token_accuracy": 0.977027119398117, + "num_tokens": 9085314.0, + "step": 3550 + }, + { + "entropy": 0.09747562363743782, + "epoch": 8.824539877300614, + "grad_norm": 0.37873855233192444, + "learning_rate": 1.8296928065522607e-05, + "loss": 0.06881467819213867, + "mean_token_accuracy": 0.9769331353902817, + "num_tokens": 9215088.0, + "step": 3600 + }, + { + "entropy": 0.09772783245891332, + "epoch": 8.94723926380368, + "grad_norm": 0.3342941105365753, + "learning_rate": 1.4731726253549227e-05, + "loss": 0.06916965007781982, + "mean_token_accuracy": 0.9772700309753418, + "num_tokens": 9342905.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2565673442397799, + "eval_loss": 0.976934015750885, + "eval_mean_token_accuracy": 0.8267382683072771, + "eval_num_tokens": 9393939.0, + "eval_runtime": 56.6695, + "eval_samples_per_second": 24.634, + "eval_steps_per_second": 3.088, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.96959540023328e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7ed08388d103b2d360274fb617b7c2ffcfd6534e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-408/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.20200841987296e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c94a7abe789084eafa44bb649296116f70ed872c --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-4080/trainer_state.json @@ -0,0 +1,954 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + }, + { + "entropy": 0.6268433964613712, + "epoch": 2.083435582822086, + "grad_norm": 0.7161462306976318, + "learning_rate": 0.0004229049328195294, + "loss": 0.5977137756347656, + "mean_token_accuracy": 0.8221012402062464, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6347901010513306, + "epoch": 2.2061349693251535, + "grad_norm": 0.717650294303894, + "learning_rate": 0.00041926568561801326, + "loss": 0.591550521850586, + "mean_token_accuracy": 0.821678705215454, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.6345827746391296, + "epoch": 2.3288343558282207, + "grad_norm": 0.6430606842041016, + "learning_rate": 0.00041526031489125996, + "loss": 0.5906002044677734, + "mean_token_accuracy": 0.8223223352432251, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6397277122735977, + "epoch": 2.4515337423312884, + "grad_norm": 0.5627941489219666, + "learning_rate": 0.0004108961490685301, + "loss": 0.6030870819091797, + "mean_token_accuracy": 0.8200687575340271, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.638802530169487, + "epoch": 2.574233128834356, + "grad_norm": 0.7276184558868408, + "learning_rate": 0.0004061811730487745, + "loss": 0.5983928298950195, + "mean_token_accuracy": 0.8206393992900849, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.6276438289880752, + "epoch": 2.6969325153374233, + "grad_norm": 0.8955866694450378, + "learning_rate": 0.00040112401359105855, + "loss": 0.5967353820800781, + "mean_token_accuracy": 0.8198855191469192, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6263330829143524, + "epoch": 2.819631901840491, + "grad_norm": 0.7027910947799683, + "learning_rate": 0.0003957339235306061, + "loss": 0.5978898620605468, + "mean_token_accuracy": 0.8205991142988205, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.6164533680677414, + "epoch": 2.942331288343558, + "grad_norm": 0.5775797963142395, + "learning_rate": 0.00039002076484934384, + "loss": 0.5849863433837891, + "mean_token_accuracy": 0.8238226908445359, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.5691366846220834, + "eval_loss": 0.6553322672843933, + "eval_mean_token_accuracy": 0.8153549589429583, + "eval_num_tokens": 3131313.0, + "eval_runtime": 56.9883, + "eval_samples_per_second": 24.496, + "eval_steps_per_second": 3.071, + "step": 1224 + }, + { + "entropy": 0.5563058633394916, + "epoch": 3.063803680981595, + "grad_norm": 0.48518672585487366, + "learning_rate": 0.00038399499063191913, + "loss": 0.5305076599121094, + "mean_token_accuracy": 0.8362203270497949, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5264888721704483, + "epoch": 3.1865030674846624, + "grad_norm": 0.5429811477661133, + "learning_rate": 0.0003776676259402065, + "loss": 0.49305004119873047, + "mean_token_accuracy": 0.8430798947811127, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5365294662117958, + "epoch": 3.30920245398773, + "grad_norm": 0.6198399066925049, + "learning_rate": 0.00037105024764129597, + "loss": 0.505305061340332, + "mean_token_accuracy": 0.8407882738113404, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5411551231145859, + "epoch": 3.4319018404907977, + "grad_norm": 0.6715341210365295, + "learning_rate": 0.00036415496322587026, + "loss": 0.5051498031616211, + "mean_token_accuracy": 0.8406214094161988, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5552197429537773, + "epoch": 3.554601226993865, + "grad_norm": 0.5532404780387878, + "learning_rate": 0.000356994388655726, + "loss": 0.5228401947021485, + "mean_token_accuracy": 0.8378301256895065, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.537499880194664, + "epoch": 3.6773006134969326, + "grad_norm": 0.5497210621833801, + "learning_rate": 0.00034958162528097133, + "loss": 0.5127570343017578, + "mean_token_accuracy": 0.841296665072441, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5413746473193168, + "epoch": 3.8, + "grad_norm": 0.5489308834075928, + "learning_rate": 0.0003419302358691309, + "loss": 0.5108597183227539, + "mean_token_accuracy": 0.8398933434486389, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5602546691894531, + "epoch": 3.9226993865030675, + "grad_norm": 0.5667650103569031, + "learning_rate": 0.00033405421979002, + "loss": 0.523597526550293, + "mean_token_accuracy": 0.8387859660387039, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5578466103758131, + "eval_loss": 0.6336107850074768, + "eval_mean_token_accuracy": 0.8219898309026445, + "eval_num_tokens": 4175084.0, + "eval_runtime": 57.1271, + "eval_samples_per_second": 24.437, + "eval_steps_per_second": 3.063, + "step": 1632 + }, + { + "entropy": 0.5006368765325258, + "epoch": 4.044171779141104, + "grad_norm": 0.5970089435577393, + "learning_rate": 0.0003259679874017874, + "loss": 0.4643829727172852, + "mean_token_accuracy": 0.8525426399828208, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.4478312811255455, + "epoch": 4.166871165644172, + "grad_norm": 0.8119876980781555, + "learning_rate": 0.0003176863336849945, + "loss": 0.40898555755615235, + "mean_token_accuracy": 0.8650373387336731, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.43789555728435514, + "epoch": 4.289570552147239, + "grad_norm": 0.5410990118980408, + "learning_rate": 0.0003092244111729683, + "loss": 0.40752113342285157, + "mean_token_accuracy": 0.8652220976352691, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4397133576869965, + "epoch": 4.412269938650307, + "grad_norm": 0.619660496711731, + "learning_rate": 0.0003005977022279593, + "loss": 0.41369033813476563, + "mean_token_accuracy": 0.8625301831960678, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4476582831144333, + "epoch": 4.534969325153375, + "grad_norm": 0.6686729192733765, + "learning_rate": 0.00029182199071382657, + "loss": 0.4207616806030273, + "mean_token_accuracy": 0.8617218750715255, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4475578209757805, + "epoch": 4.6576687116564415, + "grad_norm": 0.6534092426300049, + "learning_rate": 0.0002829133331170804, + "loss": 0.4195429229736328, + "mean_token_accuracy": 0.8632576483488082, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.45640985041856763, + "epoch": 4.780368098159509, + "grad_norm": 0.5782829523086548, + "learning_rate": 0.00027388802916912124, + "loss": 0.41994491577148435, + "mean_token_accuracy": 0.8624195390939713, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.458786401450634, + "epoch": 4.903067484662577, + "grad_norm": 0.4997099041938782, + "learning_rate": 0.00026476259202342517, + "loss": 0.4217214584350586, + "mean_token_accuracy": 0.8623241853713989, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.49301558664866857, + "eval_loss": 0.6390948295593262, + "eval_mean_token_accuracy": 0.8254796872820173, + "eval_num_tokens": 5218855.0, + "eval_runtime": 56.2663, + "eval_samples_per_second": 24.811, + "eval_steps_per_second": 3.11, + "step": 2040 + }, + { + "entropy": 0.4460540203133015, + "epoch": 5.024539877300613, + "grad_norm": 0.6123270988464355, + "learning_rate": 0.00025555371804224084, + "loss": 0.40383037567138674, + "mean_token_accuracy": 0.8667331608858976, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.34027856796979905, + "epoch": 5.147239263803681, + "grad_norm": 0.6245328187942505, + "learning_rate": 0.00024627825624807994, + "loss": 0.29572994232177735, + "mean_token_accuracy": 0.8982829064130783, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.3554780498147011, + "epoch": 5.269938650306749, + "grad_norm": 0.5149081349372864, + "learning_rate": 0.0002369531774958907, + "loss": 0.3053621292114258, + "mean_token_accuracy": 0.8955890852212905, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3499211978912353, + "epoch": 5.392638036809816, + "grad_norm": 0.5816537737846375, + "learning_rate": 0.0002275955434223227, + "loss": 0.3001568603515625, + "mean_token_accuracy": 0.8958922523260117, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3544818899035454, + "epoch": 5.515337423312883, + "grad_norm": 0.6629130244255066, + "learning_rate": 0.0002182224752288907, + "loss": 0.3112427520751953, + "mean_token_accuracy": 0.8927132803201675, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.35083883732557297, + "epoch": 5.638036809815951, + "grad_norm": 0.6243860125541687, + "learning_rate": 0.00020885112235615794, + "loss": 0.3076825332641602, + "mean_token_accuracy": 0.8949526751041412, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.3593694043159485, + "epoch": 5.7607361963190185, + "grad_norm": 0.6104142069816589, + "learning_rate": 0.00019949863110624958, + "loss": 0.3147741317749023, + "mean_token_accuracy": 0.8915749222040177, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.35077930554747583, + "epoch": 5.883435582822086, + "grad_norm": 0.5526567697525024, + "learning_rate": 0.00019018211327110994, + "loss": 0.3106169509887695, + "mean_token_accuracy": 0.8925997793674469, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.43280206646238056, + "eval_loss": 0.6834793090820312, + "eval_mean_token_accuracy": 0.8267195994513376, + "eval_num_tokens": 6262626.0, + "eval_runtime": 56.0697, + "eval_samples_per_second": 24.898, + "eval_steps_per_second": 3.121, + "step": 2448 + }, + { + "entropy": 0.35777792816210274, + "epoch": 6.004907975460123, + "grad_norm": 0.49246102571487427, + "learning_rate": 0.00018091861482390074, + "loss": 0.3066068077087402, + "mean_token_accuracy": 0.8934184981114937, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.23881351679563523, + "epoch": 6.12760736196319, + "grad_norm": 0.5304619073867798, + "learning_rate": 0.0001717250847308245, + "loss": 0.189466609954834, + "mean_token_accuracy": 0.9343969762325287, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.23922096833586692, + "epoch": 6.250306748466258, + "grad_norm": 0.5296095609664917, + "learning_rate": 0.0001626183439404375, + "loss": 0.1947933006286621, + "mean_token_accuracy": 0.9307195580005646, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.24290484443306923, + "epoch": 6.373006134969325, + "grad_norm": 0.69776451587677, + "learning_rate": 0.00015361505460718933, + "loss": 0.19909774780273437, + "mean_token_accuracy": 0.9303810286521912, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.24054641783237457, + "epoch": 6.495705521472392, + "grad_norm": 0.5774877071380615, + "learning_rate": 0.00014473168960550125, + "loss": 0.1948543930053711, + "mean_token_accuracy": 0.9311885589361191, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.24378441721200944, + "epoch": 6.61840490797546, + "grad_norm": 0.6684483289718628, + "learning_rate": 0.0001359845023901601, + "loss": 0.20006532669067384, + "mean_token_accuracy": 0.9293528574705124, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.24649206757545472, + "epoch": 6.741104294478528, + "grad_norm": 0.5416299104690552, + "learning_rate": 0.0001273894972581737, + "loss": 0.20033370971679687, + "mean_token_accuracy": 0.9294078814983368, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.245157231092453, + "epoch": 6.863803680981595, + "grad_norm": 0.6425517201423645, + "learning_rate": 0.00011896240006649818, + "loss": 0.19885097503662108, + "mean_token_accuracy": 0.929965842962265, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.23941567167639732, + "epoch": 6.986503067484662, + "grad_norm": 0.5717498660087585, + "learning_rate": 0.00011071862945921291, + "loss": 0.19223129272460937, + "mean_token_accuracy": 0.9328319728374481, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.34101752851690564, + "eval_loss": 0.7636436223983765, + "eval_mean_token_accuracy": 0.8270581153460911, + "eval_num_tokens": 7306397.0, + "eval_runtime": 57.0887, + "eval_samples_per_second": 24.453, + "eval_steps_per_second": 3.065, + "step": 2856 + }, + { + "entropy": 0.17031941931657116, + "epoch": 7.1079754601227, + "grad_norm": 0.4665500819683075, + "learning_rate": 0.0001026732686567881, + "loss": 0.1262428092956543, + "mean_token_accuracy": 0.9578677494116504, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.1474052606523037, + "epoch": 7.230674846625767, + "grad_norm": 0.45524656772613525, + "learning_rate": 9.484103785906042e-05, + "loss": 0.10753781318664551, + "mean_token_accuracy": 0.9643174183368682, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.14507727995514869, + "epoch": 7.353374233128834, + "grad_norm": 0.5161148309707642, + "learning_rate": 8.723626731240899e-05, + "loss": 0.10872675895690918, + "mean_token_accuracy": 0.9626953148841858, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.14192182645201684, + "epoch": 7.476073619631902, + "grad_norm": 0.5691169500350952, + "learning_rate": 7.987287109041075e-05, + "loss": 0.10497721672058105, + "mean_token_accuracy": 0.9646239751577377, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.14175789937376976, + "epoch": 7.598773006134969, + "grad_norm": 0.5878594517707825, + "learning_rate": 7.276432163594605e-05, + "loss": 0.10868305206298828, + "mean_token_accuracy": 0.9635398960113526, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.14344056263566018, + "epoch": 7.721472392638037, + "grad_norm": 0.5571667551994324, + "learning_rate": 6.592362511133478e-05, + "loss": 0.10832572937011718, + "mean_token_accuracy": 0.9632837051153182, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.14393038906157016, + "epoch": 7.844171779141105, + "grad_norm": 0.5589098334312439, + "learning_rate": 5.93632976016024e-05, + "loss": 0.10971030235290527, + "mean_token_accuracy": 0.9629397732019425, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.14122579149901868, + "epoch": 7.9668711656441715, + "grad_norm": 0.6026595830917358, + "learning_rate": 5.309534221441647e-05, + "loss": 0.10822047233581543, + "mean_token_accuracy": 0.9636554193496704, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.28723810638700215, + "eval_loss": 0.864037811756134, + "eval_mean_token_accuracy": 0.8260653465134757, + "eval_num_tokens": 8350168.0, + "eval_runtime": 56.6604, + "eval_samples_per_second": 24.638, + "eval_steps_per_second": 3.089, + "step": 3264 + }, + { + "entropy": 0.118123043591928, + "epoch": 8.088343558282208, + "grad_norm": 0.35258734226226807, + "learning_rate": 4.713122711859268e-05, + "loss": 0.08152069091796875, + "mean_token_accuracy": 0.9731789550395927, + "num_tokens": 8438737.0, + "step": 3300 + }, + { + "entropy": 0.10090006463229656, + "epoch": 8.211042944785277, + "grad_norm": 0.3480742573738098, + "learning_rate": 4.148186456135221e-05, + "loss": 0.06941273212432861, + "mean_token_accuracy": 0.9767383986711502, + "num_tokens": 8565559.0, + "step": 3350 + }, + { + "entropy": 0.10347712013870478, + "epoch": 8.333742331288343, + "grad_norm": 0.3159559965133667, + "learning_rate": 3.6157590902720824e-05, + "loss": 0.07076391220092773, + "mean_token_accuracy": 0.9752408146858216, + "num_tokens": 8690700.0, + "step": 3400 + }, + { + "entropy": 0.09924204669892787, + "epoch": 8.45644171779141, + "grad_norm": 0.2995770573616028, + "learning_rate": 3.1168147703600954e-05, + "loss": 0.07081368923187256, + "mean_token_accuracy": 0.9765943318605423, + "num_tokens": 8817258.0, + "step": 3450 + }, + { + "entropy": 0.09427825551480055, + "epoch": 8.579141104294479, + "grad_norm": 0.2867041826248169, + "learning_rate": 2.6522663902118405e-05, + "loss": 0.06653664588928222, + "mean_token_accuracy": 0.9782084548473358, + "num_tokens": 8954362.0, + "step": 3500 + }, + { + "entropy": 0.09728625949472189, + "epoch": 8.701840490797546, + "grad_norm": 0.24146562814712524, + "learning_rate": 2.2229639110854164e-05, + "loss": 0.06928532123565674, + "mean_token_accuracy": 0.977027119398117, + "num_tokens": 9085314.0, + "step": 3550 + }, + { + "entropy": 0.09747562363743782, + "epoch": 8.824539877300614, + "grad_norm": 0.37873855233192444, + "learning_rate": 1.8296928065522607e-05, + "loss": 0.06881467819213867, + "mean_token_accuracy": 0.9769331353902817, + "num_tokens": 9215088.0, + "step": 3600 + }, + { + "entropy": 0.09772783245891332, + "epoch": 8.94723926380368, + "grad_norm": 0.3342941105365753, + "learning_rate": 1.4731726253549227e-05, + "loss": 0.06916965007781982, + "mean_token_accuracy": 0.9772700309753418, + "num_tokens": 9342905.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.2565673442397799, + "eval_loss": 0.976934015750885, + "eval_mean_token_accuracy": 0.8267382683072771, + "eval_num_tokens": 9393939.0, + "eval_runtime": 56.6695, + "eval_samples_per_second": 24.634, + "eval_steps_per_second": 3.088, + "step": 3672 + }, + { + "entropy": 0.0916076527550967, + "epoch": 9.068711656441717, + "grad_norm": 0.23522605001926422, + "learning_rate": 1.1540556748841907e-05, + "loss": 0.06306525230407715, + "mean_token_accuracy": 0.9788417226136333, + "num_tokens": 9468612.0, + "step": 3700 + }, + { + "entropy": 0.08917598083615302, + "epoch": 9.191411042944786, + "grad_norm": 0.24837368726730347, + "learning_rate": 8.729258276844402e-06, + "loss": 0.06106118679046631, + "mean_token_accuracy": 0.9794616109132767, + "num_tokens": 9592610.0, + "step": 3750 + }, + { + "entropy": 0.08832863252609968, + "epoch": 9.314110429447853, + "grad_norm": 0.17587782442569733, + "learning_rate": 6.302974531708396e-06, + "loss": 0.06057032108306885, + "mean_token_accuracy": 0.9795139145851135, + "num_tokens": 9717974.0, + "step": 3800 + }, + { + "entropy": 0.0852269084379077, + "epoch": 9.43680981595092, + "grad_norm": 0.23637576401233673, + "learning_rate": 4.266144765130151e-06, + "loss": 0.05956338882446289, + "mean_token_accuracy": 0.9795473378896713, + "num_tokens": 9847312.0, + "step": 3850 + }, + { + "entropy": 0.08963520795106888, + "epoch": 9.559509202453988, + "grad_norm": 0.4361634850502014, + "learning_rate": 2.622495664070384e-06, + "loss": 0.062132744789123534, + "mean_token_accuracy": 0.9789100790023804, + "num_tokens": 9971026.0, + "step": 3900 + }, + { + "entropy": 0.08253341790288687, + "epoch": 9.682208588957055, + "grad_norm": 0.2019277960062027, + "learning_rate": 1.3750345322190976e-06, + "loss": 0.05810558795928955, + "mean_token_accuracy": 0.9801411455869675, + "num_tokens": 10101197.0, + "step": 3950 + }, + { + "entropy": 0.08669911231845617, + "epoch": 9.804907975460123, + "grad_norm": 0.20124948024749756, + "learning_rate": 5.260437876804016e-07, + "loss": 0.05943127632141113, + "mean_token_accuracy": 0.979077422618866, + "num_tokens": 10231249.0, + "step": 4000 + }, + { + "entropy": 0.08452129393815994, + "epoch": 9.92760736196319, + "grad_norm": 0.18919433653354645, + "learning_rate": 7.707678694451539e-08, + "loss": 0.05800063133239746, + "mean_token_accuracy": 0.9800542253255844, + "num_tokens": 10363875.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.24382151084286827, + "eval_loss": 1.0279685258865356, + "eval_mean_token_accuracy": 0.827204031263079, + "eval_num_tokens": 10437710.0, + "eval_runtime": 56.4753, + "eval_samples_per_second": 24.719, + "eval_steps_per_second": 3.099, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1884301223294848e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3aad460a78d74d84160cc6e3cb018ad4a1e7d3 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.04804280758660922, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..5498c16f14bff0b959b6ebc4e26193c8b4103fb9 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test1/checkpoint-816/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6869684666395188, + "epoch": 0.12269938650306748, + "grad_norm": 1.4528672695159912, + "learning_rate": 5.264136964343788e-05, + "loss": 1.5886520385742187, + "mean_token_accuracy": 0.6519684100151062, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9406951844692231, + "epoch": 0.24539877300613497, + "grad_norm": 1.3675724267959595, + "learning_rate": 0.00010635705295306836, + "loss": 0.8635823059082032, + "mean_token_accuracy": 0.7623954111337662, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8472799164056778, + "epoch": 0.36809815950920244, + "grad_norm": 0.848242461681366, + "learning_rate": 0.00016007273626269885, + "loss": 0.7817398834228516, + "mean_token_accuracy": 0.7812312138080597, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7993512552976608, + "epoch": 0.49079754601226994, + "grad_norm": 0.7264193892478943, + "learning_rate": 0.00021378841957232935, + "loss": 0.7406961059570313, + "mean_token_accuracy": 0.790413863658905, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7909506279230117, + "epoch": 0.6134969325153374, + "grad_norm": 0.6578923463821411, + "learning_rate": 0.00026750410288195985, + "loss": 0.7357511901855469, + "mean_token_accuracy": 0.7914552348852157, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7645951139926911, + "epoch": 0.7361963190184049, + "grad_norm": 0.844650149345398, + "learning_rate": 0.0003212197861915903, + "loss": 0.7212095642089844, + "mean_token_accuracy": 0.7942444437742233, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7707051610946656, + "epoch": 0.8588957055214724, + "grad_norm": 0.9334251284599304, + "learning_rate": 0.0003749354695012208, + "loss": 0.7167671966552734, + "mean_token_accuracy": 0.7953211319446564, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.781549910902977, + "epoch": 0.9815950920245399, + "grad_norm": 0.6472936272621155, + "learning_rate": 0.00042865115281085127, + "loss": 0.7353939819335937, + "mean_token_accuracy": 0.7930131030082702, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7875640044893537, + "eval_loss": 0.7651876211166382, + "eval_mean_token_accuracy": 0.7883888609068734, + "eval_num_tokens": 1043771.0, + "eval_runtime": 56.1402, + "eval_samples_per_second": 24.866, + "eval_steps_per_second": 3.117, + "step": 408 + }, + { + "entropy": 0.7474399413725342, + "epoch": 1.1030674846625768, + "grad_norm": 0.9284119606018066, + "learning_rate": 0.00043818515750422955, + "loss": 0.7073413848876953, + "mean_token_accuracy": 0.7961714261710041, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7287704515457153, + "epoch": 1.2257668711656442, + "grad_norm": 0.6676222085952759, + "learning_rate": 0.0004376560965874532, + "loss": 0.6876696014404297, + "mean_token_accuracy": 0.8015633362531662, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7303326517343521, + "epoch": 1.3484662576687116, + "grad_norm": 1.440736174583435, + "learning_rate": 0.0004367272641175644, + "loss": 0.6853336334228516, + "mean_token_accuracy": 0.8025881034135819, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.752444422841072, + "epoch": 1.471165644171779, + "grad_norm": 0.847322940826416, + "learning_rate": 0.00043540035953352037, + "loss": 0.7047377777099609, + "mean_token_accuracy": 0.7983024090528488, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.7128984904289246, + "epoch": 1.5938650306748468, + "grad_norm": 0.7412364482879639, + "learning_rate": 0.00043367781060719106, + "loss": 0.6685499572753906, + "mean_token_accuracy": 0.8069163882732391, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.7085144728422165, + "epoch": 1.716564417177914, + "grad_norm": 0.569710910320282, + "learning_rate": 0.0004315627690013853, + "loss": 0.6666531372070312, + "mean_token_accuracy": 0.8071046113967896, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.7012534445524216, + "epoch": 1.8392638036809816, + "grad_norm": 0.6749410629272461, + "learning_rate": 0.0004290591045034084, + "loss": 0.6621897888183593, + "mean_token_accuracy": 0.8081263571977615, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6866424483060837, + "epoch": 1.961963190184049, + "grad_norm": 0.6211123466491699, + "learning_rate": 0.0004261713979447033, + "loss": 0.6496143341064453, + "mean_token_accuracy": 0.8108037900924683, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7798067167827061, + "eval_loss": 0.6922561526298523, + "eval_mean_token_accuracy": 0.8042122350420271, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.2357, + "eval_samples_per_second": 25.274, + "eval_steps_per_second": 3.168, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.389589723338816e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dec55dd4ab4cff0540431c47af8f3f8806cb329b --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/87fz6zil) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eae1f1a87139ee1ef6876ed28f62c06c40c672fe --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1224/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1224, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.580693020417024e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6bf4ef53f450c6199ae272fa709bc1a18c45e1a7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-1632/trainer_state.json @@ -0,0 +1,398 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1632, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.779536115351488e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aa5165b6c11ba5723ed6bb595e44ab5dc62d3b4e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2040/trainer_state.json @@ -0,0 +1,489 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2040, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0964287730493312e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..cc04ca2f971efb8fdec6893cf295e334d6d5d853 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2448/trainer_state.json @@ -0,0 +1,580 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2448, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + }, + { + "entropy": 0.4233656883841813, + "epoch": 5.024539877300613, + "grad_norm": 1.176419973373413, + "learning_rate": 0.00017101670521947802, + "loss": 0.37964950561523436, + "mean_token_accuracy": 0.8736349103426693, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.3194328552484512, + "epoch": 5.147239263803681, + "grad_norm": 0.9695028066635132, + "learning_rate": 0.00016480956048459157, + "loss": 0.26651044845581057, + "mean_token_accuracy": 0.9076909011602402, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.32913370251655577, + "epoch": 5.269938650306749, + "grad_norm": 0.7606438398361206, + "learning_rate": 0.0001585692120508898, + "loss": 0.27507898330688474, + "mean_token_accuracy": 0.9048001569509506, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3264250946044922, + "epoch": 5.392638036809816, + "grad_norm": 0.7879095077514648, + "learning_rate": 0.00015230707757610745, + "loss": 0.273320255279541, + "mean_token_accuracy": 0.9042381232976914, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3339249214529991, + "epoch": 5.515337423312883, + "grad_norm": 0.9407238960266113, + "learning_rate": 0.00014603461457882375, + "loss": 0.2849284172058105, + "mean_token_accuracy": 0.9000772458314895, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.32629768148064614, + "epoch": 5.638036809815951, + "grad_norm": 0.9467188715934753, + "learning_rate": 0.0001397632994752065, + "loss": 0.28271291732788084, + "mean_token_accuracy": 0.9023476117849349, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.33983479619026186, + "epoch": 5.7607361963190185, + "grad_norm": 0.833331286907196, + "learning_rate": 0.00013350460658117884, + "loss": 0.28792747497558596, + "mean_token_accuracy": 0.8994022816419601, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.3359655112028122, + "epoch": 5.883435582822086, + "grad_norm": 0.9718000292778015, + "learning_rate": 0.00012726998711842962, + "loss": 0.28535757064819334, + "mean_token_accuracy": 0.89990447640419, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4216706611428942, + "eval_loss": 0.6848774552345276, + "eval_mean_token_accuracy": 0.828343221119472, + "eval_num_tokens": 6262626.0, + "eval_runtime": 54.603, + "eval_samples_per_second": 25.548, + "eval_steps_per_second": 3.205, + "step": 2448 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.3153170392634048e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..83b6ca0f7be377e0b89aa08b0799a8492f89cd67 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-2856/trainer_state.json @@ -0,0 +1,681 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + }, + { + "entropy": 0.4233656883841813, + "epoch": 5.024539877300613, + "grad_norm": 1.176419973373413, + "learning_rate": 0.00017101670521947802, + "loss": 0.37964950561523436, + "mean_token_accuracy": 0.8736349103426693, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.3194328552484512, + "epoch": 5.147239263803681, + "grad_norm": 0.9695028066635132, + "learning_rate": 0.00016480956048459157, + "loss": 0.26651044845581057, + "mean_token_accuracy": 0.9076909011602402, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.32913370251655577, + "epoch": 5.269938650306749, + "grad_norm": 0.7606438398361206, + "learning_rate": 0.0001585692120508898, + "loss": 0.27507898330688474, + "mean_token_accuracy": 0.9048001569509506, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3264250946044922, + "epoch": 5.392638036809816, + "grad_norm": 0.7879095077514648, + "learning_rate": 0.00015230707757610745, + "loss": 0.273320255279541, + "mean_token_accuracy": 0.9042381232976914, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3339249214529991, + "epoch": 5.515337423312883, + "grad_norm": 0.9407238960266113, + "learning_rate": 0.00014603461457882375, + "loss": 0.2849284172058105, + "mean_token_accuracy": 0.9000772458314895, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.32629768148064614, + "epoch": 5.638036809815951, + "grad_norm": 0.9467188715934753, + "learning_rate": 0.0001397632994752065, + "loss": 0.28271291732788084, + "mean_token_accuracy": 0.9023476117849349, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.33983479619026186, + "epoch": 5.7607361963190185, + "grad_norm": 0.833331286907196, + "learning_rate": 0.00013350460658117884, + "loss": 0.28792747497558596, + "mean_token_accuracy": 0.8994022816419601, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.3359655112028122, + "epoch": 5.883435582822086, + "grad_norm": 0.9718000292778015, + "learning_rate": 0.00012726998711842962, + "loss": 0.28535757064819334, + "mean_token_accuracy": 0.89990447640419, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4216706611428942, + "eval_loss": 0.6848774552345276, + "eval_mean_token_accuracy": 0.828343221119472, + "eval_num_tokens": 6262626.0, + "eval_runtime": 54.603, + "eval_samples_per_second": 25.548, + "eval_steps_per_second": 3.205, + "step": 2448 + }, + { + "entropy": 0.33349522797748293, + "epoch": 6.004907975460123, + "grad_norm": 0.6022723913192749, + "learning_rate": 0.00012107084826267792, + "loss": 0.28048072814941405, + "mean_token_accuracy": 0.9031711630146912, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.2204522843658924, + "epoch": 6.12760736196319, + "grad_norm": 0.6907380223274231, + "learning_rate": 0.00011491853227252612, + "loss": 0.16610410690307617, + "mean_token_accuracy": 0.9422257554531097, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.220889762789011, + "epoch": 6.250306748466258, + "grad_norm": 0.8346027731895447, + "learning_rate": 0.00010882429573708905, + "loss": 0.17087560653686523, + "mean_token_accuracy": 0.9396843892335892, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.2196851624548435, + "epoch": 6.373006134969325, + "grad_norm": 0.9017597436904907, + "learning_rate": 0.00010279928898036766, + "loss": 0.17283817291259765, + "mean_token_accuracy": 0.9400428485870361, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.21609510987997055, + "epoch": 6.495705521472392, + "grad_norm": 0.7931994795799255, + "learning_rate": 9.685453566005163e-05, + "loss": 0.1720712661743164, + "mean_token_accuracy": 0.9392521053552627, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.21537949055433273, + "epoch": 6.61840490797546, + "grad_norm": 0.8009449243545532, + "learning_rate": 9.100091259807638e-05, + "loss": 0.17478918075561523, + "mean_token_accuracy": 0.9384695601463318, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.21675981238484382, + "epoch": 6.741104294478528, + "grad_norm": 0.7584171295166016, + "learning_rate": 8.524912987983841e-05, + "loss": 0.17743263244628907, + "mean_token_accuracy": 0.9378731244802475, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.2188248497247696, + "epoch": 6.863803680981595, + "grad_norm": 0.8649565577507019, + "learning_rate": 7.960971125848049e-05, + "loss": 0.17595062255859376, + "mean_token_accuracy": 0.9380272418260575, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.21007086411118508, + "epoch": 6.986503067484662, + "grad_norm": 0.8617611527442932, + "learning_rate": 7.409297490009939e-05, + "loss": 0.1687315559387207, + "mean_token_accuracy": 0.9409821754693986, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.32235407480171746, + "eval_loss": 0.7729435563087463, + "eval_mean_token_accuracy": 0.828271198613303, + "eval_num_tokens": 7306397.0, + "eval_runtime": 54.7492, + "eval_samples_per_second": 25.48, + "eval_steps_per_second": 3.196, + "step": 2856 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.5332087363966208e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eb697064d71ca74a1253d31fd25ece564ba28a48 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3264/trainer_state.json @@ -0,0 +1,772 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3264, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + }, + { + "entropy": 0.4233656883841813, + "epoch": 5.024539877300613, + "grad_norm": 1.176419973373413, + "learning_rate": 0.00017101670521947802, + "loss": 0.37964950561523436, + "mean_token_accuracy": 0.8736349103426693, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.3194328552484512, + "epoch": 5.147239263803681, + "grad_norm": 0.9695028066635132, + "learning_rate": 0.00016480956048459157, + "loss": 0.26651044845581057, + "mean_token_accuracy": 0.9076909011602402, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.32913370251655577, + "epoch": 5.269938650306749, + "grad_norm": 0.7606438398361206, + "learning_rate": 0.0001585692120508898, + "loss": 0.27507898330688474, + "mean_token_accuracy": 0.9048001569509506, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3264250946044922, + "epoch": 5.392638036809816, + "grad_norm": 0.7879095077514648, + "learning_rate": 0.00015230707757610745, + "loss": 0.273320255279541, + "mean_token_accuracy": 0.9042381232976914, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3339249214529991, + "epoch": 5.515337423312883, + "grad_norm": 0.9407238960266113, + "learning_rate": 0.00014603461457882375, + "loss": 0.2849284172058105, + "mean_token_accuracy": 0.9000772458314895, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.32629768148064614, + "epoch": 5.638036809815951, + "grad_norm": 0.9467188715934753, + "learning_rate": 0.0001397632994752065, + "loss": 0.28271291732788084, + "mean_token_accuracy": 0.9023476117849349, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.33983479619026186, + "epoch": 5.7607361963190185, + "grad_norm": 0.833331286907196, + "learning_rate": 0.00013350460658117884, + "loss": 0.28792747497558596, + "mean_token_accuracy": 0.8994022816419601, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.3359655112028122, + "epoch": 5.883435582822086, + "grad_norm": 0.9718000292778015, + "learning_rate": 0.00012726998711842962, + "loss": 0.28535757064819334, + "mean_token_accuracy": 0.89990447640419, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4216706611428942, + "eval_loss": 0.6848774552345276, + "eval_mean_token_accuracy": 0.828343221119472, + "eval_num_tokens": 6262626.0, + "eval_runtime": 54.603, + "eval_samples_per_second": 25.548, + "eval_steps_per_second": 3.205, + "step": 2448 + }, + { + "entropy": 0.33349522797748293, + "epoch": 6.004907975460123, + "grad_norm": 0.6022723913192749, + "learning_rate": 0.00012107084826267792, + "loss": 0.28048072814941405, + "mean_token_accuracy": 0.9031711630146912, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.2204522843658924, + "epoch": 6.12760736196319, + "grad_norm": 0.6907380223274231, + "learning_rate": 0.00011491853227252612, + "loss": 0.16610410690307617, + "mean_token_accuracy": 0.9422257554531097, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.220889762789011, + "epoch": 6.250306748466258, + "grad_norm": 0.8346027731895447, + "learning_rate": 0.00010882429573708905, + "loss": 0.17087560653686523, + "mean_token_accuracy": 0.9396843892335892, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.2196851624548435, + "epoch": 6.373006134969325, + "grad_norm": 0.9017597436904907, + "learning_rate": 0.00010279928898036766, + "loss": 0.17283817291259765, + "mean_token_accuracy": 0.9400428485870361, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.21609510987997055, + "epoch": 6.495705521472392, + "grad_norm": 0.7931994795799255, + "learning_rate": 9.685453566005163e-05, + "loss": 0.1720712661743164, + "mean_token_accuracy": 0.9392521053552627, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.21537949055433273, + "epoch": 6.61840490797546, + "grad_norm": 0.8009449243545532, + "learning_rate": 9.100091259807638e-05, + "loss": 0.17478918075561523, + "mean_token_accuracy": 0.9384695601463318, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.21675981238484382, + "epoch": 6.741104294478528, + "grad_norm": 0.7584171295166016, + "learning_rate": 8.524912987983841e-05, + "loss": 0.17743263244628907, + "mean_token_accuracy": 0.9378731244802475, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.2188248497247696, + "epoch": 6.863803680981595, + "grad_norm": 0.8649565577507019, + "learning_rate": 7.960971125848049e-05, + "loss": 0.17595062255859376, + "mean_token_accuracy": 0.9380272418260575, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.21007086411118508, + "epoch": 6.986503067484662, + "grad_norm": 0.8617611527442932, + "learning_rate": 7.409297490009939e-05, + "loss": 0.1687315559387207, + "mean_token_accuracy": 0.9409821754693986, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.32235407480171746, + "eval_loss": 0.7729435563087463, + "eval_mean_token_accuracy": 0.828271198613303, + "eval_num_tokens": 7306397.0, + "eval_runtime": 54.7492, + "eval_samples_per_second": 25.48, + "eval_steps_per_second": 3.196, + "step": 2856 + }, + { + "entropy": 0.1553185157703631, + "epoch": 7.1079754601227, + "grad_norm": 0.7337784171104431, + "learning_rate": 6.870901450510641e-05, + "loss": 0.11502729415893555, + "mean_token_accuracy": 0.9617964462800459, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.13791010357439518, + "epoch": 7.230674846625767, + "grad_norm": 0.5963562726974487, + "learning_rate": 6.346768084028173e-05, + "loss": 0.09604933738708496, + "mean_token_accuracy": 0.9681097900867462, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.13447294190526007, + "epoch": 7.353374233128834, + "grad_norm": 0.744234561920166, + "learning_rate": 5.8378563715311995e-05, + "loss": 0.09651236534118653, + "mean_token_accuracy": 0.9678861331939698, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.13121712058782578, + "epoch": 7.476073619631902, + "grad_norm": 0.7079617381095886, + "learning_rate": 5.345097443678877e-05, + "loss": 0.09372413635253907, + "mean_token_accuracy": 0.9687917667627335, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.13528632394969464, + "epoch": 7.598773006134969, + "grad_norm": 0.7282599806785583, + "learning_rate": 4.869392877177e-05, + "loss": 0.09784893989562989, + "mean_token_accuracy": 0.9672658181190491, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.13346019081771374, + "epoch": 7.721472392638037, + "grad_norm": 0.7579892873764038, + "learning_rate": 4.411613045207589e-05, + "loss": 0.0952318286895752, + "mean_token_accuracy": 0.968282008767128, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.13583227708935738, + "epoch": 7.844171779141105, + "grad_norm": 0.6000515222549438, + "learning_rate": 3.9725955249499933e-05, + "loss": 0.09878210067749023, + "mean_token_accuracy": 0.9670963233709335, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.13274008519947528, + "epoch": 7.9668711656441715, + "grad_norm": 0.8267654776573181, + "learning_rate": 3.5531435651072355e-05, + "loss": 0.09759536743164063, + "mean_token_accuracy": 0.9673835605382919, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27422926809106557, + "eval_loss": 0.888493001461029, + "eval_mean_token_accuracy": 0.8283618112972805, + "eval_num_tokens": 8350168.0, + "eval_runtime": 54.6052, + "eval_samples_per_second": 25.547, + "eval_steps_per_second": 3.205, + "step": 3264 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.7506329083762688e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..816c6faa34dac7ce1ccc20ecc671dba9756e331f --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-3672/trainer_state.json @@ -0,0 +1,863 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3672, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + }, + { + "entropy": 0.4233656883841813, + "epoch": 5.024539877300613, + "grad_norm": 1.176419973373413, + "learning_rate": 0.00017101670521947802, + "loss": 0.37964950561523436, + "mean_token_accuracy": 0.8736349103426693, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.3194328552484512, + "epoch": 5.147239263803681, + "grad_norm": 0.9695028066635132, + "learning_rate": 0.00016480956048459157, + "loss": 0.26651044845581057, + "mean_token_accuracy": 0.9076909011602402, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.32913370251655577, + "epoch": 5.269938650306749, + "grad_norm": 0.7606438398361206, + "learning_rate": 0.0001585692120508898, + "loss": 0.27507898330688474, + "mean_token_accuracy": 0.9048001569509506, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3264250946044922, + "epoch": 5.392638036809816, + "grad_norm": 0.7879095077514648, + "learning_rate": 0.00015230707757610745, + "loss": 0.273320255279541, + "mean_token_accuracy": 0.9042381232976914, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3339249214529991, + "epoch": 5.515337423312883, + "grad_norm": 0.9407238960266113, + "learning_rate": 0.00014603461457882375, + "loss": 0.2849284172058105, + "mean_token_accuracy": 0.9000772458314895, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.32629768148064614, + "epoch": 5.638036809815951, + "grad_norm": 0.9467188715934753, + "learning_rate": 0.0001397632994752065, + "loss": 0.28271291732788084, + "mean_token_accuracy": 0.9023476117849349, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.33983479619026186, + "epoch": 5.7607361963190185, + "grad_norm": 0.833331286907196, + "learning_rate": 0.00013350460658117884, + "loss": 0.28792747497558596, + "mean_token_accuracy": 0.8994022816419601, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.3359655112028122, + "epoch": 5.883435582822086, + "grad_norm": 0.9718000292778015, + "learning_rate": 0.00012726998711842962, + "loss": 0.28535757064819334, + "mean_token_accuracy": 0.89990447640419, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4216706611428942, + "eval_loss": 0.6848774552345276, + "eval_mean_token_accuracy": 0.828343221119472, + "eval_num_tokens": 6262626.0, + "eval_runtime": 54.603, + "eval_samples_per_second": 25.548, + "eval_steps_per_second": 3.205, + "step": 2448 + }, + { + "entropy": 0.33349522797748293, + "epoch": 6.004907975460123, + "grad_norm": 0.6022723913192749, + "learning_rate": 0.00012107084826267792, + "loss": 0.28048072814941405, + "mean_token_accuracy": 0.9031711630146912, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.2204522843658924, + "epoch": 6.12760736196319, + "grad_norm": 0.6907380223274231, + "learning_rate": 0.00011491853227252612, + "loss": 0.16610410690307617, + "mean_token_accuracy": 0.9422257554531097, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.220889762789011, + "epoch": 6.250306748466258, + "grad_norm": 0.8346027731895447, + "learning_rate": 0.00010882429573708905, + "loss": 0.17087560653686523, + "mean_token_accuracy": 0.9396843892335892, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.2196851624548435, + "epoch": 6.373006134969325, + "grad_norm": 0.9017597436904907, + "learning_rate": 0.00010279928898036766, + "loss": 0.17283817291259765, + "mean_token_accuracy": 0.9400428485870361, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.21609510987997055, + "epoch": 6.495705521472392, + "grad_norm": 0.7931994795799255, + "learning_rate": 9.685453566005163e-05, + "loss": 0.1720712661743164, + "mean_token_accuracy": 0.9392521053552627, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.21537949055433273, + "epoch": 6.61840490797546, + "grad_norm": 0.8009449243545532, + "learning_rate": 9.100091259807638e-05, + "loss": 0.17478918075561523, + "mean_token_accuracy": 0.9384695601463318, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.21675981238484382, + "epoch": 6.741104294478528, + "grad_norm": 0.7584171295166016, + "learning_rate": 8.524912987983841e-05, + "loss": 0.17743263244628907, + "mean_token_accuracy": 0.9378731244802475, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.2188248497247696, + "epoch": 6.863803680981595, + "grad_norm": 0.8649565577507019, + "learning_rate": 7.960971125848049e-05, + "loss": 0.17595062255859376, + "mean_token_accuracy": 0.9380272418260575, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.21007086411118508, + "epoch": 6.986503067484662, + "grad_norm": 0.8617611527442932, + "learning_rate": 7.409297490009939e-05, + "loss": 0.1687315559387207, + "mean_token_accuracy": 0.9409821754693986, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.32235407480171746, + "eval_loss": 0.7729435563087463, + "eval_mean_token_accuracy": 0.828271198613303, + "eval_num_tokens": 7306397.0, + "eval_runtime": 54.7492, + "eval_samples_per_second": 25.48, + "eval_steps_per_second": 3.196, + "step": 2856 + }, + { + "entropy": 0.1553185157703631, + "epoch": 7.1079754601227, + "grad_norm": 0.7337784171104431, + "learning_rate": 6.870901450510641e-05, + "loss": 0.11502729415893555, + "mean_token_accuracy": 0.9617964462800459, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.13791010357439518, + "epoch": 7.230674846625767, + "grad_norm": 0.5963562726974487, + "learning_rate": 6.346768084028173e-05, + "loss": 0.09604933738708496, + "mean_token_accuracy": 0.9681097900867462, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.13447294190526007, + "epoch": 7.353374233128834, + "grad_norm": 0.744234561920166, + "learning_rate": 5.8378563715311995e-05, + "loss": 0.09651236534118653, + "mean_token_accuracy": 0.9678861331939698, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.13121712058782578, + "epoch": 7.476073619631902, + "grad_norm": 0.7079617381095886, + "learning_rate": 5.345097443678877e-05, + "loss": 0.09372413635253907, + "mean_token_accuracy": 0.9687917667627335, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.13528632394969464, + "epoch": 7.598773006134969, + "grad_norm": 0.7282599806785583, + "learning_rate": 4.869392877177e-05, + "loss": 0.09784893989562989, + "mean_token_accuracy": 0.9672658181190491, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.13346019081771374, + "epoch": 7.721472392638037, + "grad_norm": 0.7579892873764038, + "learning_rate": 4.411613045207589e-05, + "loss": 0.0952318286895752, + "mean_token_accuracy": 0.968282008767128, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.13583227708935738, + "epoch": 7.844171779141105, + "grad_norm": 0.6000515222549438, + "learning_rate": 3.9725955249499933e-05, + "loss": 0.09878210067749023, + "mean_token_accuracy": 0.9670963233709335, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.13274008519947528, + "epoch": 7.9668711656441715, + "grad_norm": 0.8267654776573181, + "learning_rate": 3.5531435651072355e-05, + "loss": 0.09759536743164063, + "mean_token_accuracy": 0.9673835605382919, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27422926809106557, + "eval_loss": 0.888493001461029, + "eval_mean_token_accuracy": 0.8283618112972805, + "eval_num_tokens": 8350168.0, + "eval_runtime": 54.6052, + "eval_samples_per_second": 25.547, + "eval_steps_per_second": 3.205, + "step": 3264 + }, + { + "entropy": 0.11354313956366645, + "epoch": 8.088343558282208, + "grad_norm": 0.5075483918190002, + "learning_rate": 3.154024616241484e-05, + "loss": 0.07540683746337891, + "mean_token_accuracy": 0.9749541607770053, + "num_tokens": 8438737.0, + "step": 3300 + }, + { + "entropy": 0.09892106257379055, + "epoch": 8.211042944785277, + "grad_norm": 0.351868599653244, + "learning_rate": 2.7759689266076296e-05, + "loss": 0.0670144510269165, + "mean_token_accuracy": 0.9774316263198852, + "num_tokens": 8565559.0, + "step": 3350 + }, + { + "entropy": 0.10244134511798621, + "epoch": 8.333742331288343, + "grad_norm": 0.4751427173614502, + "learning_rate": 2.4196682060540395e-05, + "loss": 0.06803131580352784, + "mean_token_accuracy": 0.9764371293783188, + "num_tokens": 8690700.0, + "step": 3400 + }, + { + "entropy": 0.09730133015662432, + "epoch": 8.45644171779141, + "grad_norm": 0.49166423082351685, + "learning_rate": 2.0857743604351807e-05, + "loss": 0.06794160842895508, + "mean_token_accuracy": 0.9773359906673431, + "num_tokens": 8817258.0, + "step": 3450 + }, + { + "entropy": 0.0930729215592146, + "epoch": 8.579141104294479, + "grad_norm": 0.46982425451278687, + "learning_rate": 1.7748982988516496e-05, + "loss": 0.06399593353271485, + "mean_token_accuracy": 0.9787030357122422, + "num_tokens": 8954362.0, + "step": 3500 + }, + { + "entropy": 0.09571650367230176, + "epoch": 8.701840490797546, + "grad_norm": 0.4293615520000458, + "learning_rate": 1.4876088158998911e-05, + "loss": 0.06665454864501953, + "mean_token_accuracy": 0.977586322426796, + "num_tokens": 9085314.0, + "step": 3550 + }, + { + "entropy": 0.09772399775683879, + "epoch": 8.824539877300614, + "grad_norm": 0.36890819668769836, + "learning_rate": 1.2244315509767943e-05, + "loss": 0.06794494152069092, + "mean_token_accuracy": 0.977256600856781, + "num_tokens": 9215088.0, + "step": 3600 + }, + { + "entropy": 0.09747388873249292, + "epoch": 8.94723926380368, + "grad_norm": 0.40667724609375, + "learning_rate": 9.858480265432268e-06, + "loss": 0.06607833385467529, + "mean_token_accuracy": 0.9779314315319061, + "num_tokens": 9342905.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24860519809382303, + "eval_loss": 0.9853287935256958, + "eval_mean_token_accuracy": 0.8290910639081682, + "eval_num_tokens": 9393939.0, + "eval_runtime": 54.1258, + "eval_samples_per_second": 25.773, + "eval_steps_per_second": 3.233, + "step": 3672 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.96959540023328e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..04092290a01f9c7728372badae0c9c9fca26acd2 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-408/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 408, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.20200841987296e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6589f4001c5c28fe3d181e727d7e766db98eecf6 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-4080/trainer_state.json @@ -0,0 +1,954 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + }, + { + "entropy": 0.6131738848758467, + "epoch": 2.083435582822086, + "grad_norm": 1.272260308265686, + "learning_rate": 0.0002830082410301935, + "loss": 0.5861021804809571, + "mean_token_accuracy": 0.8247757692529698, + "num_tokens": 2177128.0, + "step": 850 + }, + { + "entropy": 0.6047256541252136, + "epoch": 2.2061349693251535, + "grad_norm": 0.9789379239082336, + "learning_rate": 0.00028057285456565526, + "loss": 0.5819529342651367, + "mean_token_accuracy": 0.8245759695768357, + "num_tokens": 2299605.0, + "step": 900 + }, + { + "entropy": 0.592314181625843, + "epoch": 2.3288343558282207, + "grad_norm": 1.1999729871749878, + "learning_rate": 0.0002778924580129482, + "loss": 0.5756857299804687, + "mean_token_accuracy": 0.8254415565729141, + "num_tokens": 2430751.0, + "step": 950 + }, + { + "entropy": 0.6082978844642639, + "epoch": 2.4515337423312884, + "grad_norm": 0.8222051858901978, + "learning_rate": 0.0002749719555614387, + "loss": 0.5864879989624023, + "mean_token_accuracy": 0.8237437629699706, + "num_tokens": 2553654.0, + "step": 1000 + }, + { + "entropy": 0.5973288315534592, + "epoch": 2.574233128834356, + "grad_norm": 0.9686626195907593, + "learning_rate": 0.0002718166907104137, + "loss": 0.5792468643188476, + "mean_token_accuracy": 0.825029706954956, + "num_tokens": 2680735.0, + "step": 1050 + }, + { + "entropy": 0.591679208278656, + "epoch": 2.6969325153374233, + "grad_norm": 0.7853561043739319, + "learning_rate": 0.000268432436492343, + "loss": 0.582892837524414, + "mean_token_accuracy": 0.8242555004358292, + "num_tokens": 2813796.0, + "step": 1100 + }, + { + "entropy": 0.6070335671305657, + "epoch": 2.819631901840491, + "grad_norm": 0.97103351354599, + "learning_rate": 0.0002648253849102468, + "loss": 0.5842210388183594, + "mean_token_accuracy": 0.8242060190439224, + "num_tokens": 2940312.0, + "step": 1150 + }, + { + "entropy": 0.5928181883692741, + "epoch": 2.942331288343558, + "grad_norm": 0.821699321269989, + "learning_rate": 0.0002610021356084932, + "loss": 0.5727723693847656, + "mean_token_accuracy": 0.826916481256485, + "num_tokens": 3070898.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.590432653938021, + "eval_loss": 0.6449909210205078, + "eval_mean_token_accuracy": 0.8177921431405204, + "eval_num_tokens": 3131313.0, + "eval_runtime": 55.3145, + "eval_samples_per_second": 25.219, + "eval_steps_per_second": 3.164, + "step": 1224 + }, + { + "entropy": 0.5483075320118606, + "epoch": 3.063803680981595, + "grad_norm": 0.7041144371032715, + "learning_rate": 0.0002569696837977545, + "loss": 0.5140520477294922, + "mean_token_accuracy": 0.8396318085265883, + "num_tokens": 3201120.0, + "step": 1250 + }, + { + "entropy": 0.5025479900836944, + "epoch": 3.1865030674846624, + "grad_norm": 0.8355836868286133, + "learning_rate": 0.00025273540745621484, + "loss": 0.47410289764404295, + "mean_token_accuracy": 0.8482581508159638, + "num_tokens": 3325619.0, + "step": 1300 + }, + { + "entropy": 0.5020234334468842, + "epoch": 3.30920245398773, + "grad_norm": 0.9736531376838684, + "learning_rate": 0.000248307053830448, + "loss": 0.4891999816894531, + "mean_token_accuracy": 0.8448011082410812, + "num_tokens": 3455554.0, + "step": 1350 + }, + { + "entropy": 0.5080654540657997, + "epoch": 3.4319018404907977, + "grad_norm": 1.0834203958511353, + "learning_rate": 0.0002436927252606621, + "loss": 0.4924654006958008, + "mean_token_accuracy": 0.844255143404007, + "num_tokens": 3580945.0, + "step": 1400 + }, + { + "entropy": 0.5200119104981422, + "epoch": 3.554601226993865, + "grad_norm": 0.8291606307029724, + "learning_rate": 0.00023890086435624734, + "loss": 0.5029202651977539, + "mean_token_accuracy": 0.8424944174289704, + "num_tokens": 3703139.0, + "step": 1450 + }, + { + "entropy": 0.5124201083183288, + "epoch": 3.6773006134969326, + "grad_norm": 0.8587970733642578, + "learning_rate": 0.00023394023854875025, + "loss": 0.4949214553833008, + "mean_token_accuracy": 0.8452944302558899, + "num_tokens": 3833407.0, + "step": 1500 + }, + { + "entropy": 0.5084337800741195, + "epoch": 3.8, + "grad_norm": 1.0057843923568726, + "learning_rate": 0.0002288199240505364, + "loss": 0.49095394134521486, + "mean_token_accuracy": 0.8456550347805023, + "num_tokens": 3964068.0, + "step": 1550 + }, + { + "entropy": 0.5280214902758599, + "epoch": 3.9226993865030675, + "grad_norm": 0.7453445792198181, + "learning_rate": 0.00022354928924849234, + "loss": 0.5079964447021484, + "mean_token_accuracy": 0.8413498830795288, + "num_tokens": 4091244.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5494849445138659, + "eval_loss": 0.6249175667762756, + "eval_mean_token_accuracy": 0.8237050444739206, + "eval_num_tokens": 4175084.0, + "eval_runtime": 56.1739, + "eval_samples_per_second": 24.834, + "eval_steps_per_second": 3.115, + "step": 1632 + }, + { + "entropy": 0.46954305605454877, + "epoch": 4.044171779141104, + "grad_norm": 0.710486650466919, + "learning_rate": 0.0002181379775631504, + "loss": 0.44171287536621096, + "mean_token_accuracy": 0.8586679510395936, + "num_tokens": 4220924.0, + "step": 1650 + }, + { + "entropy": 0.41737838625907897, + "epoch": 4.166871165644172, + "grad_norm": 1.1825510263442993, + "learning_rate": 0.00021259588980459754, + "loss": 0.37950809478759767, + "mean_token_accuracy": 0.87354656457901, + "num_tokens": 4342912.0, + "step": 1700 + }, + { + "entropy": 0.41023619323968885, + "epoch": 4.289570552147239, + "grad_norm": 0.8626420497894287, + "learning_rate": 0.00020693316605745158, + "loss": 0.3841147613525391, + "mean_token_accuracy": 0.8725562179088593, + "num_tokens": 4475673.0, + "step": 1750 + }, + { + "entropy": 0.4168246221542358, + "epoch": 4.412269938650307, + "grad_norm": 0.9204051494598389, + "learning_rate": 0.00020116016712804847, + "loss": 0.38734291076660154, + "mean_token_accuracy": 0.8706799060106277, + "num_tokens": 4605538.0, + "step": 1800 + }, + { + "entropy": 0.4218893802165985, + "epoch": 4.534969325153375, + "grad_norm": 0.9835762977600098, + "learning_rate": 0.0001952874555877861, + "loss": 0.39813385009765623, + "mean_token_accuracy": 0.8677069127559662, + "num_tokens": 4730588.0, + "step": 1850 + }, + { + "entropy": 0.4191485980153084, + "epoch": 4.6576687116564415, + "grad_norm": 0.9436285495758057, + "learning_rate": 0.00018932577644730818, + "loss": 0.3958011245727539, + "mean_token_accuracy": 0.8692897647619248, + "num_tokens": 4856517.0, + "step": 1900 + }, + { + "entropy": 0.42448077738285067, + "epoch": 4.780368098159509, + "grad_norm": 1.047423005104065, + "learning_rate": 0.00018328603749688839, + "loss": 0.3967106246948242, + "mean_token_accuracy": 0.8681233793497085, + "num_tokens": 4988951.0, + "step": 1950 + }, + { + "entropy": 0.4255312702059746, + "epoch": 4.903067484662577, + "grad_norm": 0.6979464292526245, + "learning_rate": 0.0001771792893489846, + "loss": 0.397794189453125, + "mean_token_accuracy": 0.8682954198122025, + "num_tokens": 5119329.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.48526300924164906, + "eval_loss": 0.6391404867172241, + "eval_mean_token_accuracy": 0.8266797767366681, + "eval_num_tokens": 5218855.0, + "eval_runtime": 54.4313, + "eval_samples_per_second": 25.629, + "eval_steps_per_second": 3.215, + "step": 2040 + }, + { + "entropy": 0.4233656883841813, + "epoch": 5.024539877300613, + "grad_norm": 1.176419973373413, + "learning_rate": 0.00017101670521947802, + "loss": 0.37964950561523436, + "mean_token_accuracy": 0.8736349103426693, + "num_tokens": 5244358.0, + "step": 2050 + }, + { + "entropy": 0.3194328552484512, + "epoch": 5.147239263803681, + "grad_norm": 0.9695028066635132, + "learning_rate": 0.00016480956048459157, + "loss": 0.26651044845581057, + "mean_token_accuracy": 0.9076909011602402, + "num_tokens": 5376529.0, + "step": 2100 + }, + { + "entropy": 0.32913370251655577, + "epoch": 5.269938650306749, + "grad_norm": 0.7606438398361206, + "learning_rate": 0.0001585692120508898, + "loss": 0.27507898330688474, + "mean_token_accuracy": 0.9048001569509506, + "num_tokens": 5502665.0, + "step": 2150 + }, + { + "entropy": 0.3264250946044922, + "epoch": 5.392638036809816, + "grad_norm": 0.7879095077514648, + "learning_rate": 0.00015230707757610745, + "loss": 0.273320255279541, + "mean_token_accuracy": 0.9042381232976914, + "num_tokens": 5635833.0, + "step": 2200 + }, + { + "entropy": 0.3339249214529991, + "epoch": 5.515337423312883, + "grad_norm": 0.9407238960266113, + "learning_rate": 0.00014603461457882375, + "loss": 0.2849284172058105, + "mean_token_accuracy": 0.9000772458314895, + "num_tokens": 5761306.0, + "step": 2250 + }, + { + "entropy": 0.32629768148064614, + "epoch": 5.638036809815951, + "grad_norm": 0.9467188715934753, + "learning_rate": 0.0001397632994752065, + "loss": 0.28271291732788084, + "mean_token_accuracy": 0.9023476117849349, + "num_tokens": 5886926.0, + "step": 2300 + }, + { + "entropy": 0.33983479619026186, + "epoch": 5.7607361963190185, + "grad_norm": 0.833331286907196, + "learning_rate": 0.00013350460658117884, + "loss": 0.28792747497558596, + "mean_token_accuracy": 0.8994022816419601, + "num_tokens": 6016346.0, + "step": 2350 + }, + { + "entropy": 0.3359655112028122, + "epoch": 5.883435582822086, + "grad_norm": 0.9718000292778015, + "learning_rate": 0.00012726998711842962, + "loss": 0.28535757064819334, + "mean_token_accuracy": 0.89990447640419, + "num_tokens": 6144634.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.4216706611428942, + "eval_loss": 0.6848774552345276, + "eval_mean_token_accuracy": 0.828343221119472, + "eval_num_tokens": 6262626.0, + "eval_runtime": 54.603, + "eval_samples_per_second": 25.548, + "eval_steps_per_second": 3.205, + "step": 2448 + }, + { + "entropy": 0.33349522797748293, + "epoch": 6.004907975460123, + "grad_norm": 0.6022723913192749, + "learning_rate": 0.00012107084826267792, + "loss": 0.28048072814941405, + "mean_token_accuracy": 0.9031711630146912, + "num_tokens": 6267831.0, + "step": 2450 + }, + { + "entropy": 0.2204522843658924, + "epoch": 6.12760736196319, + "grad_norm": 0.6907380223274231, + "learning_rate": 0.00011491853227252612, + "loss": 0.16610410690307617, + "mean_token_accuracy": 0.9422257554531097, + "num_tokens": 6399002.0, + "step": 2500 + }, + { + "entropy": 0.220889762789011, + "epoch": 6.250306748466258, + "grad_norm": 0.8346027731895447, + "learning_rate": 0.00010882429573708905, + "loss": 0.17087560653686523, + "mean_token_accuracy": 0.9396843892335892, + "num_tokens": 6530675.0, + "step": 2550 + }, + { + "entropy": 0.2196851624548435, + "epoch": 6.373006134969325, + "grad_norm": 0.9017597436904907, + "learning_rate": 0.00010279928898036766, + "loss": 0.17283817291259765, + "mean_token_accuracy": 0.9400428485870361, + "num_tokens": 6656178.0, + "step": 2600 + }, + { + "entropy": 0.21609510987997055, + "epoch": 6.495705521472392, + "grad_norm": 0.7931994795799255, + "learning_rate": 9.685453566005163e-05, + "loss": 0.1720712661743164, + "mean_token_accuracy": 0.9392521053552627, + "num_tokens": 6786838.0, + "step": 2650 + }, + { + "entropy": 0.21537949055433273, + "epoch": 6.61840490797546, + "grad_norm": 0.8009449243545532, + "learning_rate": 9.100091259807638e-05, + "loss": 0.17478918075561523, + "mean_token_accuracy": 0.9384695601463318, + "num_tokens": 6912627.0, + "step": 2700 + }, + { + "entropy": 0.21675981238484382, + "epoch": 6.741104294478528, + "grad_norm": 0.7584171295166016, + "learning_rate": 8.524912987983841e-05, + "loss": 0.17743263244628907, + "mean_token_accuracy": 0.9378731244802475, + "num_tokens": 7037952.0, + "step": 2750 + }, + { + "entropy": 0.2188248497247696, + "epoch": 6.863803680981595, + "grad_norm": 0.8649565577507019, + "learning_rate": 7.960971125848049e-05, + "loss": 0.17595062255859376, + "mean_token_accuracy": 0.9380272418260575, + "num_tokens": 7161727.0, + "step": 2800 + }, + { + "entropy": 0.21007086411118508, + "epoch": 6.986503067484662, + "grad_norm": 0.8617611527442932, + "learning_rate": 7.409297490009939e-05, + "loss": 0.1687315559387207, + "mean_token_accuracy": 0.9409821754693986, + "num_tokens": 7293830.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.32235407480171746, + "eval_loss": 0.7729435563087463, + "eval_mean_token_accuracy": 0.828271198613303, + "eval_num_tokens": 7306397.0, + "eval_runtime": 54.7492, + "eval_samples_per_second": 25.48, + "eval_steps_per_second": 3.196, + "step": 2856 + }, + { + "entropy": 0.1553185157703631, + "epoch": 7.1079754601227, + "grad_norm": 0.7337784171104431, + "learning_rate": 6.870901450510641e-05, + "loss": 0.11502729415893555, + "mean_token_accuracy": 0.9617964462800459, + "num_tokens": 7412985.0, + "step": 2900 + }, + { + "entropy": 0.13791010357439518, + "epoch": 7.230674846625767, + "grad_norm": 0.5963562726974487, + "learning_rate": 6.346768084028173e-05, + "loss": 0.09604933738708496, + "mean_token_accuracy": 0.9681097900867462, + "num_tokens": 7543754.0, + "step": 2950 + }, + { + "entropy": 0.13447294190526007, + "epoch": 7.353374233128834, + "grad_norm": 0.744234561920166, + "learning_rate": 5.8378563715311995e-05, + "loss": 0.09651236534118653, + "mean_token_accuracy": 0.9678861331939698, + "num_tokens": 7673840.0, + "step": 3000 + }, + { + "entropy": 0.13121712058782578, + "epoch": 7.476073619631902, + "grad_norm": 0.7079617381095886, + "learning_rate": 5.345097443678877e-05, + "loss": 0.09372413635253907, + "mean_token_accuracy": 0.9687917667627335, + "num_tokens": 7807664.0, + "step": 3050 + }, + { + "entropy": 0.13528632394969464, + "epoch": 7.598773006134969, + "grad_norm": 0.7282599806785583, + "learning_rate": 4.869392877177e-05, + "loss": 0.09784893989562989, + "mean_token_accuracy": 0.9672658181190491, + "num_tokens": 7933830.0, + "step": 3100 + }, + { + "entropy": 0.13346019081771374, + "epoch": 7.721472392638037, + "grad_norm": 0.7579892873764038, + "learning_rate": 4.411613045207589e-05, + "loss": 0.0952318286895752, + "mean_token_accuracy": 0.968282008767128, + "num_tokens": 8063545.0, + "step": 3150 + }, + { + "entropy": 0.13583227708935738, + "epoch": 7.844171779141105, + "grad_norm": 0.6000515222549438, + "learning_rate": 3.9725955249499933e-05, + "loss": 0.09878210067749023, + "mean_token_accuracy": 0.9670963233709335, + "num_tokens": 8189862.0, + "step": 3200 + }, + { + "entropy": 0.13274008519947528, + "epoch": 7.9668711656441715, + "grad_norm": 0.8267654776573181, + "learning_rate": 3.5531435651072355e-05, + "loss": 0.09759536743164063, + "mean_token_accuracy": 0.9673835605382919, + "num_tokens": 8316465.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.27422926809106557, + "eval_loss": 0.888493001461029, + "eval_mean_token_accuracy": 0.8283618112972805, + "eval_num_tokens": 8350168.0, + "eval_runtime": 54.6052, + "eval_samples_per_second": 25.547, + "eval_steps_per_second": 3.205, + "step": 3264 + }, + { + "entropy": 0.11354313956366645, + "epoch": 8.088343558282208, + "grad_norm": 0.5075483918190002, + "learning_rate": 3.154024616241484e-05, + "loss": 0.07540683746337891, + "mean_token_accuracy": 0.9749541607770053, + "num_tokens": 8438737.0, + "step": 3300 + }, + { + "entropy": 0.09892106257379055, + "epoch": 8.211042944785277, + "grad_norm": 0.351868599653244, + "learning_rate": 2.7759689266076296e-05, + "loss": 0.0670144510269165, + "mean_token_accuracy": 0.9774316263198852, + "num_tokens": 8565559.0, + "step": 3350 + }, + { + "entropy": 0.10244134511798621, + "epoch": 8.333742331288343, + "grad_norm": 0.4751427173614502, + "learning_rate": 2.4196682060540395e-05, + "loss": 0.06803131580352784, + "mean_token_accuracy": 0.9764371293783188, + "num_tokens": 8690700.0, + "step": 3400 + }, + { + "entropy": 0.09730133015662432, + "epoch": 8.45644171779141, + "grad_norm": 0.49166423082351685, + "learning_rate": 2.0857743604351807e-05, + "loss": 0.06794160842895508, + "mean_token_accuracy": 0.9773359906673431, + "num_tokens": 8817258.0, + "step": 3450 + }, + { + "entropy": 0.0930729215592146, + "epoch": 8.579141104294479, + "grad_norm": 0.46982425451278687, + "learning_rate": 1.7748982988516496e-05, + "loss": 0.06399593353271485, + "mean_token_accuracy": 0.9787030357122422, + "num_tokens": 8954362.0, + "step": 3500 + }, + { + "entropy": 0.09571650367230176, + "epoch": 8.701840490797546, + "grad_norm": 0.4293615520000458, + "learning_rate": 1.4876088158998911e-05, + "loss": 0.06665454864501953, + "mean_token_accuracy": 0.977586322426796, + "num_tokens": 9085314.0, + "step": 3550 + }, + { + "entropy": 0.09772399775683879, + "epoch": 8.824539877300614, + "grad_norm": 0.36890819668769836, + "learning_rate": 1.2244315509767943e-05, + "loss": 0.06794494152069092, + "mean_token_accuracy": 0.977256600856781, + "num_tokens": 9215088.0, + "step": 3600 + }, + { + "entropy": 0.09747388873249292, + "epoch": 8.94723926380368, + "grad_norm": 0.40667724609375, + "learning_rate": 9.858480265432268e-06, + "loss": 0.06607833385467529, + "mean_token_accuracy": 0.9779314315319061, + "num_tokens": 9342905.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.24860519809382303, + "eval_loss": 0.9853287935256958, + "eval_mean_token_accuracy": 0.8290910639081682, + "eval_num_tokens": 9393939.0, + "eval_runtime": 54.1258, + "eval_samples_per_second": 25.773, + "eval_steps_per_second": 3.233, + "step": 3672 + }, + { + "entropy": 0.09252041284785126, + "epoch": 9.068711656441717, + "grad_norm": 0.3194105625152588, + "learning_rate": 7.722947671061197e-06, + "loss": 0.06175976276397705, + "mean_token_accuracy": 0.9793706885491958, + "num_tokens": 9468612.0, + "step": 3700 + }, + { + "entropy": 0.091169836781919, + "epoch": 9.191411042944786, + "grad_norm": 0.3423891067504883, + "learning_rate": 5.841625005311143e-06, + "loss": 0.059695353507995604, + "mean_token_accuracy": 0.979728177189827, + "num_tokens": 9592610.0, + "step": 3750 + }, + { + "entropy": 0.08912075750529766, + "epoch": 9.314110429447853, + "grad_norm": 0.2739139497280121, + "learning_rate": 4.217954431470577e-06, + "loss": 0.05969821453094482, + "mean_token_accuracy": 0.979958308339119, + "num_tokens": 9717974.0, + "step": 3800 + }, + { + "entropy": 0.08561772257089614, + "epoch": 9.43680981595092, + "grad_norm": 0.2996142506599426, + "learning_rate": 2.8549066995037362e-06, + "loss": 0.05884579658508301, + "mean_token_accuracy": 0.9794363099336624, + "num_tokens": 9847312.0, + "step": 3850 + }, + { + "entropy": 0.09101765561848879, + "epoch": 9.559509202453988, + "grad_norm": 0.28566721081733704, + "learning_rate": 1.7549757106157715e-06, + "loss": 0.06139932155609131, + "mean_token_accuracy": 0.978882754445076, + "num_tokens": 9971026.0, + "step": 3900 + }, + { + "entropy": 0.08350372072309256, + "epoch": 9.682208588957055, + "grad_norm": 0.2542772889137268, + "learning_rate": 9.201739542848183e-07, + "loss": 0.05746087074279785, + "mean_token_accuracy": 0.980304804444313, + "num_tokens": 10101197.0, + "step": 3950 + }, + { + "entropy": 0.08740887437015772, + "epoch": 9.804907975460123, + "grad_norm": 0.24912101030349731, + "learning_rate": 3.520288261092993e-07, + "loss": 0.058651576042175295, + "mean_token_accuracy": 0.9794082939624786, + "num_tokens": 10231249.0, + "step": 4000 + }, + { + "entropy": 0.08617342811077833, + "epoch": 9.92760736196319, + "grad_norm": 0.25340721011161804, + "learning_rate": 5.157983320741952e-08, + "loss": 0.05704680919647217, + "mean_token_accuracy": 0.9801818627119064, + "num_tokens": 10363875.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.2382178349154336, + "eval_loss": 1.029752492904663, + "eval_mean_token_accuracy": 0.8286988289015633, + "eval_num_tokens": 10437710.0, + "eval_runtime": 53.9582, + "eval_samples_per_second": 25.853, + "eval_steps_per_second": 3.243, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1884301223294848e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..c8c5873f84d69cbffcfabea1f03ac6cb67d10e5e --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 256, + "lora_bias": false, + "lora_dropout": 0.05723170086707252, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "o_proj", + "v_proj", + "up_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ac59e07b69b9724169dae55b372264277f5d905a --- /dev/null +++ b/random_original_Estonian/Qwen3.5-2B-Base_random_splits_original_features_train_random_splits_original_features_test2/checkpoint-816/trainer_state.json @@ -0,0 +1,216 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.6333012944459915, + "epoch": 0.12269938650306748, + "grad_norm": 2.5465779304504395, + "learning_rate": 3.52276369274868e-05, + "loss": 1.53487060546875, + "mean_token_accuracy": 0.6598937398195267, + "num_tokens": 129823.0, + "step": 50 + }, + { + "entropy": 0.9258983463048935, + "epoch": 0.24539877300613497, + "grad_norm": 2.258607864379883, + "learning_rate": 7.117420522084068e-05, + "loss": 0.85067626953125, + "mean_token_accuracy": 0.7652804219722747, + "num_tokens": 253466.0, + "step": 100 + }, + { + "entropy": 0.8346361368894577, + "epoch": 0.36809815950920244, + "grad_norm": 1.417574167251587, + "learning_rate": 0.00010712077351419455, + "loss": 0.7736109161376953, + "mean_token_accuracy": 0.7826414209604263, + "num_tokens": 387484.0, + "step": 150 + }, + { + "entropy": 0.7951875519752503, + "epoch": 0.49079754601226994, + "grad_norm": 1.0634689331054688, + "learning_rate": 0.00014306734180754844, + "loss": 0.7422314453125, + "mean_token_accuracy": 0.7903646248579025, + "num_tokens": 513501.0, + "step": 200 + }, + { + "entropy": 0.7834344464540481, + "epoch": 0.6134969325153374, + "grad_norm": 0.9857470393180847, + "learning_rate": 0.00017901391010090233, + "loss": 0.733411865234375, + "mean_token_accuracy": 0.7923515325784684, + "num_tokens": 643515.0, + "step": 250 + }, + { + "entropy": 0.7621527630090713, + "epoch": 0.7361963190184049, + "grad_norm": 1.1431680917739868, + "learning_rate": 0.00021496047839425616, + "loss": 0.7220622253417969, + "mean_token_accuracy": 0.793820441365242, + "num_tokens": 772238.0, + "step": 300 + }, + { + "entropy": 0.7626819384098052, + "epoch": 0.8588957055214724, + "grad_norm": 1.2470793724060059, + "learning_rate": 0.0002509070466876101, + "loss": 0.7197529602050782, + "mean_token_accuracy": 0.7959642672538757, + "num_tokens": 899105.0, + "step": 350 + }, + { + "entropy": 0.7625831609964371, + "epoch": 0.9815950920245399, + "grad_norm": 1.1210194826126099, + "learning_rate": 0.00028685361498096394, + "loss": 0.7235118103027344, + "mean_token_accuracy": 0.7958246481418609, + "num_tokens": 1025860.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8266113938604083, + "eval_loss": 0.7417611479759216, + "eval_mean_token_accuracy": 0.792023388998849, + "eval_num_tokens": 1043771.0, + "eval_runtime": 55.7939, + "eval_samples_per_second": 25.003, + "eval_steps_per_second": 3.137, + "step": 408 + }, + { + "entropy": 0.7339564622050584, + "epoch": 1.1030674846625768, + "grad_norm": 1.6268937587738037, + "learning_rate": 0.0002932337767829499, + "loss": 0.6926748657226562, + "mean_token_accuracy": 0.7993869944052263, + "num_tokens": 1158723.0, + "step": 450 + }, + { + "entropy": 0.7134076035022736, + "epoch": 1.2257668711656442, + "grad_norm": 1.2319769859313965, + "learning_rate": 0.0002928797288921947, + "loss": 0.67477294921875, + "mean_token_accuracy": 0.8043139892816543, + "num_tokens": 1289372.0, + "step": 500 + }, + { + "entropy": 0.7083246171474457, + "epoch": 1.3484662576687116, + "grad_norm": 1.5780729055404663, + "learning_rate": 0.000292258153632331, + "loss": 0.6769942474365235, + "mean_token_accuracy": 0.8059876197576523, + "num_tokens": 1413943.0, + "step": 550 + }, + { + "entropy": 0.7342139089107513, + "epoch": 1.471165644171779, + "grad_norm": 1.1391289234161377, + "learning_rate": 0.0002913701882689536, + "loss": 0.6926898193359375, + "mean_token_accuracy": 0.8011569637060165, + "num_tokens": 1536890.0, + "step": 600 + }, + { + "entropy": 0.6911397916078568, + "epoch": 1.5938650306748468, + "grad_norm": 1.184077501296997, + "learning_rate": 0.00029021745746849034, + "loss": 0.6618209075927735, + "mean_token_accuracy": 0.8077511537075043, + "num_tokens": 1663992.0, + "step": 650 + }, + { + "entropy": 0.6756490755081177, + "epoch": 1.716564417177914, + "grad_norm": 0.8195196986198425, + "learning_rate": 0.00028880207032563047, + "loss": 0.6586660766601562, + "mean_token_accuracy": 0.8085329169034958, + "num_tokens": 1791126.0, + "step": 700 + }, + { + "entropy": 0.6688846790790558, + "epoch": 1.8392638036809816, + "grad_norm": 1.0433928966522217, + "learning_rate": 0.0002871266165044178, + "loss": 0.6558318328857422, + "mean_token_accuracy": 0.809660274386406, + "num_tokens": 1921148.0, + "step": 750 + }, + { + "entropy": 0.6700717681646347, + "epoch": 1.961963190184049, + "grad_norm": 0.7902872562408447, + "learning_rate": 0.00028519416150007, + "loss": 0.6427510833740234, + "mean_token_accuracy": 0.8128749799728393, + "num_tokens": 2049795.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7883352865491594, + "eval_loss": 0.6791115999221802, + "eval_mean_token_accuracy": 0.8063773039409092, + "eval_num_tokens": 2087542.0, + "eval_runtime": 55.3057, + "eval_samples_per_second": 25.223, + "eval_steps_per_second": 3.164, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.389589723338816e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-408/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b9596096c71017d49955136a4e23d9f97138d305 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.08602048083239339, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "gate_proj", + "up_proj", + "o_proj", + "down_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..d1b167ab783f7c80ec3cdb3282d86815c3b0d8ff --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-4080/trainer_state.json @@ -0,0 +1,1064 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4080, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.2408065378665922, + "epoch": 0.12269938650306748, + "grad_norm": 1.6718658208847046, + "learning_rate": 4.487804970912036e-05, + "loss": 2.129624786376953, + "mean_token_accuracy": 0.5743644836544991, + "num_tokens": 134329.0, + "step": 50 + }, + { + "entropy": 1.1078504550457, + "epoch": 0.24539877300613497, + "grad_norm": 2.4091813564300537, + "learning_rate": 9.067197798373296e-05, + "loss": 1.0346390533447265, + "mean_token_accuracy": 0.7085381114482879, + "num_tokens": 262407.0, + "step": 100 + }, + { + "entropy": 0.9421598988771439, + "epoch": 0.36809815950920244, + "grad_norm": 1.3913646936416626, + "learning_rate": 0.00013646590625834558, + "loss": 0.8716551208496094, + "mean_token_accuracy": 0.7449040985107422, + "num_tokens": 401136.0, + "step": 150 + }, + { + "entropy": 0.8723550146818161, + "epoch": 0.49079754601226994, + "grad_norm": 1.1664468050003052, + "learning_rate": 0.0001822598345329582, + "loss": 0.8095977783203125, + "mean_token_accuracy": 0.7585240352153778, + "num_tokens": 531647.0, + "step": 200 + }, + { + "entropy": 0.8466519421339035, + "epoch": 0.6134969325153374, + "grad_norm": 1.0145421028137207, + "learning_rate": 0.00022805376280757083, + "loss": 0.7830857849121093, + "mean_token_accuracy": 0.7661705195903779, + "num_tokens": 666127.0, + "step": 250 + }, + { + "entropy": 0.8180295366048813, + "epoch": 0.7361963190184049, + "grad_norm": 0.9731705784797668, + "learning_rate": 0.00027384769108218336, + "loss": 0.7573603820800782, + "mean_token_accuracy": 0.7704453605413437, + "num_tokens": 799292.0, + "step": 300 + }, + { + "entropy": 0.7990227991342544, + "epoch": 0.8588957055214724, + "grad_norm": 1.0642374753952026, + "learning_rate": 0.000319641619356796, + "loss": 0.73709228515625, + "mean_token_accuracy": 0.7752733880281448, + "num_tokens": 930614.0, + "step": 350 + }, + { + "entropy": 0.7899087685346603, + "epoch": 0.9815950920245399, + "grad_norm": 0.880613386631012, + "learning_rate": 0.0003654355476314086, + "loss": 0.7347319793701171, + "mean_token_accuracy": 0.7766731631755829, + "num_tokens": 1061654.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.83522472347532, + "eval_mean_token_accuracy": 0.7484098076820374, + "eval_not_syn_loss": 0.7872886061668396, + "eval_not_syn_runtime": 54.8697, + "eval_not_syn_samples_per_second": 25.424, + "eval_not_syn_steps_per_second": 3.189, + "eval_num_tokens": 1080196.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7784442271505083, + "eval_mean_token_accuracy": 0.8021791185651507, + "eval_num_tokens": 1080196.0, + "eval_syn_loss": 0.6980098485946655, + "eval_syn_runtime": 56.6342, + "eval_syn_samples_per_second": 24.632, + "eval_syn_steps_per_second": 3.09, + "step": 408 + }, + { + "entropy": 0.75672573694075, + "epoch": 1.1030674846625768, + "grad_norm": 0.9244198203086853, + "learning_rate": 0.00037356351883456205, + "loss": 0.7048745727539063, + "mean_token_accuracy": 0.783016896609104, + "num_tokens": 1199170.0, + "step": 450 + }, + { + "entropy": 0.7406797379255294, + "epoch": 1.2257668711656442, + "grad_norm": 0.6744896769523621, + "learning_rate": 0.00037311248151765587, + "loss": 0.6860730743408203, + "mean_token_accuracy": 0.787369327545166, + "num_tokens": 1334298.0, + "step": 500 + }, + { + "entropy": 0.7424973237514496, + "epoch": 1.3484662576687116, + "grad_norm": 0.92305588722229, + "learning_rate": 0.000372320629215228, + "loss": 0.6855724334716797, + "mean_token_accuracy": 0.7871134179830551, + "num_tokens": 1463189.0, + "step": 550 + }, + { + "entropy": 0.7502243828773498, + "epoch": 1.471165644171779, + "grad_norm": 0.8259047269821167, + "learning_rate": 0.00037118941074037944, + "loss": 0.6949834442138672, + "mean_token_accuracy": 0.7851914083957672, + "num_tokens": 1590310.0, + "step": 600 + }, + { + "entropy": 0.7233260518312454, + "epoch": 1.5938650306748468, + "grad_norm": 0.882722795009613, + "learning_rate": 0.00036972089582775814, + "loss": 0.6649005889892579, + "mean_token_accuracy": 0.7898879665136337, + "num_tokens": 1721836.0, + "step": 650 + }, + { + "entropy": 0.7162708967924118, + "epoch": 1.716564417177914, + "grad_norm": 0.6593027114868164, + "learning_rate": 0.0003679177713466678, + "loss": 0.6653710174560546, + "mean_token_accuracy": 0.7905678844451904, + "num_tokens": 1853507.0, + "step": 700 + }, + { + "entropy": 0.7150520020723343, + "epoch": 1.8392638036809816, + "grad_norm": 0.665340781211853, + "learning_rate": 0.0003657833363850354, + "loss": 0.6600938415527344, + "mean_token_accuracy": 0.7918649202585221, + "num_tokens": 1988113.0, + "step": 750 + }, + { + "entropy": 0.6993386310338974, + "epoch": 1.961963190184049, + "grad_norm": 0.6502187252044678, + "learning_rate": 0.00036332149621323294, + "loss": 0.6469607543945313, + "mean_token_accuracy": 0.7950778317451477, + "num_tokens": 2121314.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7315139893123082, + "eval_mean_token_accuracy": 0.7713504004478454, + "eval_not_syn_loss": 0.699893593788147, + "eval_not_syn_runtime": 55.4135, + "eval_not_syn_samples_per_second": 25.174, + "eval_not_syn_steps_per_second": 3.158, + "eval_num_tokens": 2160392.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6848451798302787, + "eval_mean_token_accuracy": 0.814895657471248, + "eval_num_tokens": 2160392.0, + "eval_syn_loss": 0.6415576934814453, + "eval_syn_runtime": 56.4735, + "eval_syn_samples_per_second": 24.702, + "eval_syn_steps_per_second": 3.099, + "step": 816 + }, + { + "entropy": 0.6596363644407253, + "epoch": 2.083435582822086, + "grad_norm": 0.6647700071334839, + "learning_rate": 0.00036053675513879666, + "loss": 0.599151611328125, + "mean_token_accuracy": 0.8071986864311527, + "num_tokens": 2253179.0, + "step": 850 + }, + { + "entropy": 0.6626429110765457, + "epoch": 2.2061349693251535, + "grad_norm": 0.7356868386268616, + "learning_rate": 0.00035743420826511783, + "loss": 0.6004195404052735, + "mean_token_accuracy": 0.805362731218338, + "num_tokens": 2379965.0, + "step": 900 + }, + { + "entropy": 0.6503470712900161, + "epoch": 2.3288343558282207, + "grad_norm": 1.095608115196228, + "learning_rate": 0.0003540195321691835, + "loss": 0.5951333999633789, + "mean_token_accuracy": 0.807324025630951, + "num_tokens": 2515593.0, + "step": 950 + }, + { + "entropy": 0.660577870607376, + "epoch": 2.4515337423312884, + "grad_norm": 0.6602179408073425, + "learning_rate": 0.00035029897451542325, + "loss": 0.6056365585327148, + "mean_token_accuracy": 0.8057438349723816, + "num_tokens": 2642790.0, + "step": 1000 + }, + { + "entropy": 0.6555853113532066, + "epoch": 2.574233128834356, + "grad_norm": 0.7459151148796082, + "learning_rate": 0.00034627934262466606, + "loss": 0.5963204193115235, + "mean_token_accuracy": 0.8060598099231719, + "num_tokens": 2774396.0, + "step": 1050 + }, + { + "entropy": 0.6505853188037872, + "epoch": 2.6969325153374233, + "grad_norm": 0.6359792947769165, + "learning_rate": 0.00034196799101912103, + "loss": 0.5993065643310547, + "mean_token_accuracy": 0.806280387043953, + "num_tokens": 2912102.0, + "step": 1100 + }, + { + "entropy": 0.6509970253705979, + "epoch": 2.819631901840491, + "grad_norm": 0.8086174726486206, + "learning_rate": 0.0003373728079661707, + "loss": 0.6001268005371094, + "mean_token_accuracy": 0.806866540312767, + "num_tokens": 3042987.0, + "step": 1150 + }, + { + "entropy": 0.6404844307899475, + "epoch": 2.942331288343558, + "grad_norm": 0.7038407325744629, + "learning_rate": 0.0003325022010455975, + "loss": 0.5870526123046875, + "mean_token_accuracy": 0.8104586935043335, + "num_tokens": 3178044.0, + "step": 1200 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6645996727262224, + "eval_mean_token_accuracy": 0.798654237134116, + "eval_not_syn_loss": 0.6586771011352539, + "eval_not_syn_runtime": 55.1088, + "eval_not_syn_samples_per_second": 25.314, + "eval_not_syn_steps_per_second": 3.176, + "eval_num_tokens": 3240588.0, + "step": 1224 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6233607809884207, + "eval_mean_token_accuracy": 0.7977249867575509, + "eval_num_tokens": 3240588.0, + "eval_syn_loss": 0.6288298964500427, + "eval_syn_runtime": 56.6081, + "eval_syn_samples_per_second": 24.643, + "eval_syn_steps_per_second": 3.091, + "step": 1224 + }, + { + "entropy": 0.6081961104364106, + "epoch": 3.063803680981595, + "grad_norm": 0.6161203980445862, + "learning_rate": 0.0003273650817666497, + "loss": 0.5481716537475586, + "mean_token_accuracy": 0.8175542035488167, + "num_tokens": 3312797.0, + "step": 1250 + }, + { + "entropy": 0.5877064895629883, + "epoch": 3.1865030674846624, + "grad_norm": 0.6909636855125427, + "learning_rate": 0.00032197084926309234, + "loss": 0.5224150466918945, + "mean_token_accuracy": 0.8250984263420105, + "num_tokens": 3441629.0, + "step": 1300 + }, + { + "entropy": 0.5952656552195549, + "epoch": 3.30920245398773, + "grad_norm": 0.8147026300430298, + "learning_rate": 0.0003163293730960758, + "loss": 0.5349474334716797, + "mean_token_accuracy": 0.8213064837455749, + "num_tokens": 3576093.0, + "step": 1350 + }, + { + "entropy": 0.5937123569846153, + "epoch": 3.4319018404907977, + "grad_norm": 0.8690970540046692, + "learning_rate": 0.0003104509751962869, + "loss": 0.5310481262207031, + "mean_token_accuracy": 0.8229135835170746, + "num_tokens": 3705912.0, + "step": 1400 + }, + { + "entropy": 0.6067730465531349, + "epoch": 3.554601226993865, + "grad_norm": 0.7433330416679382, + "learning_rate": 0.0003043464109784207, + "loss": 0.5450045394897461, + "mean_token_accuracy": 0.8205251175165177, + "num_tokens": 3832376.0, + "step": 1450 + }, + { + "entropy": 0.5926914718747139, + "epoch": 3.6773006134969326, + "grad_norm": 0.7265587449073792, + "learning_rate": 0.00029802684966252974, + "loss": 0.5330168914794922, + "mean_token_accuracy": 0.821294127702713, + "num_tokens": 3967273.0, + "step": 1500 + }, + { + "entropy": 0.5881199997663498, + "epoch": 3.8, + "grad_norm": 0.7681817412376404, + "learning_rate": 0.00029150385383825187, + "loss": 0.5288929367065429, + "mean_token_accuracy": 0.8234161508083343, + "num_tokens": 4102434.0, + "step": 1550 + }, + { + "entropy": 0.6008686199784279, + "epoch": 3.9226993865030675, + "grad_norm": 0.9384056329727173, + "learning_rate": 0.0002847893583093113, + "loss": 0.5437511444091797, + "mean_token_accuracy": 0.8201366275548935, + "num_tokens": 4234037.0, + "step": 1600 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6218148783275059, + "eval_mean_token_accuracy": 0.783796865599496, + "eval_not_syn_loss": 0.6558170318603516, + "eval_not_syn_runtime": 55.429, + "eval_not_syn_samples_per_second": 25.167, + "eval_not_syn_steps_per_second": 3.157, + "eval_num_tokens": 4320784.0, + "step": 1632 + }, + { + "epoch": 4.0, + "eval_entropy": 0.5833754854542869, + "eval_mean_token_accuracy": 0.8278223422595433, + "eval_num_tokens": 4320784.0, + "eval_syn_loss": 0.604770302772522, + "eval_syn_runtime": 56.9593, + "eval_syn_samples_per_second": 24.491, + "eval_syn_steps_per_second": 3.072, + "step": 1632 + }, + { + "entropy": 0.559690699733869, + "epoch": 4.044171779141104, + "grad_norm": 0.65739905834198, + "learning_rate": 0.0002778956482569963, + "loss": 0.49702674865722657, + "mean_token_accuracy": 0.8317034166268628, + "num_tokens": 4368268.0, + "step": 1650 + }, + { + "entropy": 0.5401161375641823, + "epoch": 4.166871165644172, + "grad_norm": 1.5176353454589844, + "learning_rate": 0.0002708353367625691, + "loss": 0.4598746109008789, + "mean_token_accuracy": 0.8412033343315124, + "num_tokens": 4494525.0, + "step": 1700 + }, + { + "entropy": 0.5299916860461235, + "epoch": 4.289570552147239, + "grad_norm": 0.685965359210968, + "learning_rate": 0.0002636213417297333, + "loss": 0.45663665771484374, + "mean_token_accuracy": 0.8412413811683654, + "num_tokens": 4631985.0, + "step": 1750 + }, + { + "entropy": 0.5352088099718094, + "epoch": 4.412269938650307, + "grad_norm": 0.8471158742904663, + "learning_rate": 0.0002562668622493824, + "loss": 0.4625707626342773, + "mean_token_accuracy": 0.8400991547107697, + "num_tokens": 4766323.0, + "step": 1800 + }, + { + "entropy": 0.5425067806243896, + "epoch": 4.534969325153375, + "grad_norm": 0.8595437407493591, + "learning_rate": 0.00024878535444987474, + "loss": 0.47124954223632814, + "mean_token_accuracy": 0.8376108628511428, + "num_tokens": 4895606.0, + "step": 1850 + }, + { + "entropy": 0.5419312778115273, + "epoch": 4.6576687116564415, + "grad_norm": 0.8436959981918335, + "learning_rate": 0.00024119050687701818, + "loss": 0.4708108901977539, + "mean_token_accuracy": 0.8390797019004822, + "num_tokens": 5025912.0, + "step": 1900 + }, + { + "entropy": 0.5392271488904953, + "epoch": 4.780368098159509, + "grad_norm": 0.8691944479942322, + "learning_rate": 0.00023349621544881404, + "loss": 0.46682029724121094, + "mean_token_accuracy": 0.8385172665119172, + "num_tokens": 5162972.0, + "step": 1950 + }, + { + "entropy": 0.5332251599431038, + "epoch": 4.903067484662577, + "grad_norm": 0.6814598441123962, + "learning_rate": 0.00022571655803078078, + "loss": 0.4687493896484375, + "mean_token_accuracy": 0.8388861840963364, + "num_tokens": 5297935.0, + "step": 2000 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5847777877535139, + "eval_mean_token_accuracy": 0.7782382583618164, + "eval_not_syn_loss": 0.6678160429000854, + "eval_not_syn_runtime": 55.1467, + "eval_not_syn_samples_per_second": 25.296, + "eval_not_syn_steps_per_second": 3.173, + "eval_num_tokens": 5400980.0, + "step": 2040 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5505962218557086, + "eval_mean_token_accuracy": 0.8308270008223397, + "eval_num_tokens": 5400980.0, + "eval_syn_loss": 0.606867253780365, + "eval_syn_runtime": 56.9768, + "eval_syn_samples_per_second": 24.484, + "eval_syn_steps_per_second": 3.071, + "step": 2040 + }, + { + "entropy": 0.533086326688227, + "epoch": 5.024539877300613, + "grad_norm": 0.9453939199447632, + "learning_rate": 0.00021786576867837776, + "loss": 0.45802555084228513, + "mean_token_accuracy": 0.8429379854539428, + "num_tokens": 5427362.0, + "step": 2050 + }, + { + "entropy": 0.46721911430358887, + "epoch": 5.147239263803681, + "grad_norm": 0.9237910509109497, + "learning_rate": 0.00020995821159365647, + "loss": 0.37914146423339845, + "mean_token_accuracy": 0.8648965793848038, + "num_tokens": 5564186.0, + "step": 2100 + }, + { + "entropy": 0.4682381117343903, + "epoch": 5.269938650306749, + "grad_norm": 0.7424130439758301, + "learning_rate": 0.0002020083548437879, + "loss": 0.3859384536743164, + "mean_token_accuracy": 0.8622810471057892, + "num_tokens": 5694728.0, + "step": 2150 + }, + { + "entropy": 0.464998793900013, + "epoch": 5.392638036809816, + "grad_norm": 0.9224902987480164, + "learning_rate": 0.0001940307438895544, + "loss": 0.3839593505859375, + "mean_token_accuracy": 0.8627353543043137, + "num_tokens": 5832581.0, + "step": 2200 + }, + { + "entropy": 0.4753906837105751, + "epoch": 5.515337423312883, + "grad_norm": 0.8931427001953125, + "learning_rate": 0.0001860399749722367, + "loss": 0.393004264831543, + "mean_token_accuracy": 0.8586806547641754, + "num_tokens": 5962417.0, + "step": 2250 + }, + { + "entropy": 0.4671635267138481, + "epoch": 5.638036809815951, + "grad_norm": 0.9559172987937927, + "learning_rate": 0.00017805066840759193, + "loss": 0.38852767944335936, + "mean_token_accuracy": 0.8615026396512985, + "num_tokens": 6092338.0, + "step": 2300 + }, + { + "entropy": 0.4746100226044655, + "epoch": 5.7607361963190185, + "grad_norm": 0.9070253968238831, + "learning_rate": 0.00017007744183578254, + "loss": 0.3960166168212891, + "mean_token_accuracy": 0.8591635781526565, + "num_tokens": 6226327.0, + "step": 2350 + }, + { + "entropy": 0.4698092398047447, + "epoch": 5.883435582822086, + "grad_norm": 0.8354232907295227, + "learning_rate": 0.0001621348834762011, + "loss": 0.39321571350097656, + "mean_token_accuracy": 0.8603039687871933, + "num_tokens": 6359083.0, + "step": 2400 + }, + { + "epoch": 6.0, + "eval_entropy": 0.536405998979296, + "eval_mean_token_accuracy": 0.790239520413535, + "eval_not_syn_loss": 0.6884409785270691, + "eval_not_syn_runtime": 54.9927, + "eval_not_syn_samples_per_second": 25.367, + "eval_not_syn_steps_per_second": 3.182, + "eval_num_tokens": 6481176.0, + "step": 2448 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5039791010107313, + "eval_mean_token_accuracy": 0.8163431504794529, + "eval_num_tokens": 6481176.0, + "eval_syn_loss": 0.6377155184745789, + "eval_syn_runtime": 57.196, + "eval_syn_samples_per_second": 24.39, + "eval_syn_steps_per_second": 3.06, + "step": 2448 + }, + { + "entropy": 0.4671963176341972, + "epoch": 6.004907975460123, + "grad_norm": 0.8445424437522888, + "learning_rate": 0.00015423752543612516, + "loss": 0.3872825241088867, + "mean_token_accuracy": 0.8618519083418027, + "num_tokens": 6486549.0, + "step": 2450 + }, + { + "entropy": 0.3890652832388878, + "epoch": 6.12760736196319, + "grad_norm": 0.725812554359436, + "learning_rate": 0.00014639981712203683, + "loss": 0.2923568153381348, + "mean_token_accuracy": 0.8928290349245072, + "num_tokens": 6622384.0, + "step": 2500 + }, + { + "entropy": 0.39649384289979933, + "epoch": 6.250306748466258, + "grad_norm": 0.9143152236938477, + "learning_rate": 0.000138636098802257, + "loss": 0.303057861328125, + "mean_token_accuracy": 0.8888710218667984, + "num_tokens": 6758637.0, + "step": 2550 + }, + { + "entropy": 0.4020465177297592, + "epoch": 6.373006134969325, + "grad_norm": 0.9407342076301575, + "learning_rate": 0.00013096057536926304, + "loss": 0.3073894691467285, + "mean_token_accuracy": 0.8894575881958008, + "num_tokens": 6888533.0, + "step": 2600 + }, + { + "entropy": 0.397317076921463, + "epoch": 6.495705521472392, + "grad_norm": 0.8517777323722839, + "learning_rate": 0.00012338729034969828, + "loss": 0.30508209228515626, + "mean_token_accuracy": 0.8886856293678284, + "num_tokens": 7023729.0, + "step": 2650 + }, + { + "entropy": 0.3915065750479698, + "epoch": 6.61840490797546, + "grad_norm": 0.862319827079773, + "learning_rate": 0.00011593010020962378, + "loss": 0.30237462997436526, + "mean_token_accuracy": 0.8897094148397445, + "num_tokens": 7153821.0, + "step": 2700 + }, + { + "entropy": 0.3977497002482414, + "epoch": 6.741104294478528, + "grad_norm": 0.8821370601654053, + "learning_rate": 0.00010860264900202563, + "loss": 0.3067088508605957, + "mean_token_accuracy": 0.8873285222053527, + "num_tokens": 7283513.0, + "step": 2750 + }, + { + "entropy": 0.4002616259455681, + "epoch": 6.863803680981595, + "grad_norm": 1.069462776184082, + "learning_rate": 0.0001014183434029644, + "loss": 0.30735193252563475, + "mean_token_accuracy": 0.8880347841978073, + "num_tokens": 7411578.0, + "step": 2800 + }, + { + "entropy": 0.3951660317182541, + "epoch": 6.986503067484662, + "grad_norm": 0.8845367431640625, + "learning_rate": 9.439032818204105e-05, + "loss": 0.30523622512817383, + "mean_token_accuracy": 0.8894886714220047, + "num_tokens": 7548356.0, + "step": 2850 + }, + { + "epoch": 7.0, + "eval_entropy": 0.4834100874832698, + "eval_mean_token_accuracy": 0.788761088166918, + "eval_not_syn_loss": 0.7379753589630127, + "eval_not_syn_runtime": 55.6309, + "eval_not_syn_samples_per_second": 25.076, + "eval_not_syn_steps_per_second": 3.146, + "eval_num_tokens": 7561372.0, + "step": 2856 + }, + { + "epoch": 7.0, + "eval_entropy": 0.453818735906056, + "eval_mean_token_accuracy": 0.8209771479879107, + "eval_num_tokens": 7561372.0, + "eval_syn_loss": 0.6787521243095398, + "eval_syn_runtime": 56.9966, + "eval_syn_samples_per_second": 24.475, + "eval_syn_steps_per_second": 3.07, + "step": 2856 + }, + { + "entropy": 0.3386827930055483, + "epoch": 7.1079754601227, + "grad_norm": 1.0214653015136719, + "learning_rate": 8.753146215206042e-05, + "loss": 0.2361087417602539, + "mean_token_accuracy": 0.9141153067049353, + "num_tokens": 7671657.0, + "step": 2900 + }, + { + "entropy": 0.31863692075014116, + "epoch": 7.230674846625767, + "grad_norm": 0.9141477346420288, + "learning_rate": 8.085429464189586e-05, + "loss": 0.21914518356323243, + "mean_token_accuracy": 0.9194145703315735, + "num_tokens": 7806989.0, + "step": 2950 + }, + { + "entropy": 0.31718266174197196, + "epoch": 7.353374233128834, + "grad_norm": 0.9127821922302246, + "learning_rate": 7.437104253560078e-05, + "loss": 0.22112932205200195, + "mean_token_accuracy": 0.9190003037452698, + "num_tokens": 7941760.0, + "step": 3000 + }, + { + "entropy": 0.3143314927816391, + "epoch": 7.476073619631902, + "grad_norm": 1.1398990154266357, + "learning_rate": 6.809356791977874e-05, + "loss": 0.2191557502746582, + "mean_token_accuracy": 0.918931410908699, + "num_tokens": 8080328.0, + "step": 3050 + }, + { + "entropy": 0.32102111548185347, + "epoch": 7.598773006134969, + "grad_norm": 1.069332480430603, + "learning_rate": 6.203335638010852e-05, + "loss": 0.22624496459960938, + "mean_token_accuracy": 0.9173947256803513, + "num_tokens": 8210831.0, + "step": 3100 + }, + { + "entropy": 0.31791565626859664, + "epoch": 7.721472392638037, + "grad_norm": 1.1536624431610107, + "learning_rate": 5.62014959867348e-05, + "loss": 0.22338006973266603, + "mean_token_accuracy": 0.9177713346481323, + "num_tokens": 8344985.0, + "step": 3150 + }, + { + "entropy": 0.3209437537193298, + "epoch": 7.844171779141105, + "grad_norm": 1.1040980815887451, + "learning_rate": 5.06086570069728e-05, + "loss": 0.2263322639465332, + "mean_token_accuracy": 0.9170012313127518, + "num_tokens": 8475716.0, + "step": 3200 + }, + { + "entropy": 0.31527657777071, + "epoch": 7.9668711656441715, + "grad_norm": 1.1415972709655762, + "learning_rate": 4.526507238244653e-05, + "loss": 0.2216329574584961, + "mean_token_accuracy": 0.918340762257576, + "num_tokens": 8606709.0, + "step": 3250 + }, + { + "epoch": 8.0, + "eval_entropy": 0.43602277857916694, + "eval_mean_token_accuracy": 0.7907402624402727, + "eval_not_syn_loss": 0.8163036704063416, + "eval_not_syn_runtime": 55.3868, + "eval_not_syn_samples_per_second": 25.187, + "eval_not_syn_steps_per_second": 3.16, + "eval_num_tokens": 8641568.0, + "step": 3264 + }, + { + "epoch": 8.0, + "eval_entropy": 0.4093018649305616, + "eval_mean_token_accuracy": 0.8147984709058489, + "eval_num_tokens": 8641568.0, + "eval_syn_loss": 0.7571278810501099, + "eval_syn_runtime": 56.797, + "eval_syn_samples_per_second": 24.561, + "eval_syn_steps_per_second": 3.081, + "step": 3264 + }, + { + "entropy": 0.2847619162063406, + "epoch": 8.088343558282208, + "grad_norm": 1.0268603563308716, + "learning_rate": 4.018051900638024e-05, + "loss": 0.1806371307373047, + "mean_token_accuracy": 0.9346237525795446, + "num_tokens": 8733256.0, + "step": 3300 + }, + { + "entropy": 0.26448936730623246, + "epoch": 8.211042944785277, + "grad_norm": 1.307308554649353, + "learning_rate": 3.536429983529935e-05, + "loss": 0.16438600540161133, + "mean_token_accuracy": 0.9407236540317535, + "num_tokens": 8864507.0, + "step": 3350 + }, + { + "entropy": 0.26411333337426185, + "epoch": 8.333742331288343, + "grad_norm": 1.0053174495697021, + "learning_rate": 3.082522686786935e-05, + "loss": 0.16243309020996094, + "mean_token_accuracy": 0.9401588958501815, + "num_tokens": 8993933.0, + "step": 3400 + }, + { + "entropy": 0.2589409455657005, + "epoch": 8.45644171779141, + "grad_norm": 1.1849397420883179, + "learning_rate": 2.6571605022016655e-05, + "loss": 0.1622679328918457, + "mean_token_accuracy": 0.9411727702617645, + "num_tokens": 9124819.0, + "step": 3450 + }, + { + "entropy": 0.25455976352095605, + "epoch": 8.579141104294479, + "grad_norm": 0.9689339995384216, + "learning_rate": 2.2611216939830125e-05, + "loss": 0.16137266159057617, + "mean_token_accuracy": 0.941931157708168, + "num_tokens": 9266865.0, + "step": 3500 + }, + { + "entropy": 0.25809177130460736, + "epoch": 8.701840490797546, + "grad_norm": 1.097219467163086, + "learning_rate": 1.8951308748044324e-05, + "loss": 0.16276371002197265, + "mean_token_accuracy": 0.9418226885795593, + "num_tokens": 9402291.0, + "step": 3550 + }, + { + "entropy": 0.2551534166932106, + "epoch": 8.824539877300614, + "grad_norm": 1.1789699792861938, + "learning_rate": 1.5598576800158975e-05, + "loss": 0.16089740753173828, + "mean_token_accuracy": 0.942240217924118, + "num_tokens": 9536676.0, + "step": 3600 + }, + { + "entropy": 0.25731841623783114, + "epoch": 8.94723926380368, + "grad_norm": 1.071371078491211, + "learning_rate": 1.2559155424451429e-05, + "loss": 0.16048816680908204, + "mean_token_accuracy": 0.9426142275333405, + "num_tokens": 9668941.0, + "step": 3650 + }, + { + "epoch": 9.0, + "eval_entropy": 0.4000720078604562, + "eval_mean_token_accuracy": 0.791837249483381, + "eval_not_syn_loss": 0.9112135171890259, + "eval_not_syn_runtime": 55.748, + "eval_not_syn_samples_per_second": 25.023, + "eval_not_syn_steps_per_second": 3.139, + "eval_num_tokens": 9721764.0, + "step": 3672 + }, + { + "epoch": 9.0, + "eval_entropy": 0.3756853074686868, + "eval_mean_token_accuracy": 0.8066899194036211, + "eval_num_tokens": 9721764.0, + "eval_syn_loss": 0.8509045243263245, + "eval_syn_runtime": 56.7641, + "eval_syn_samples_per_second": 24.575, + "eval_syn_steps_per_second": 3.083, + "step": 3672 + }, + { + "entropy": 0.24660127993785974, + "epoch": 9.068711656441717, + "grad_norm": 1.2988486289978027, + "learning_rate": 9.838605700298561e-06, + "loss": 0.1471036720275879, + "mean_token_accuracy": 0.948410421308845, + "num_tokens": 9799022.0, + "step": 3700 + }, + { + "entropy": 0.23661178544163705, + "epoch": 9.191411042944786, + "grad_norm": 1.1723612546920776, + "learning_rate": 7.4419052833441635e-06, + "loss": 0.13447829246520995, + "mean_token_accuracy": 0.9532408624887466, + "num_tokens": 9927323.0, + "step": 3750 + }, + { + "entropy": 0.23699646830558777, + "epoch": 9.314110429447853, + "grad_norm": 0.9691436290740967, + "learning_rate": 5.373439298127954e-06, + "loss": 0.13515523910522462, + "mean_token_accuracy": 0.9527407556772232, + "num_tokens": 10057034.0, + "step": 3800 + }, + { + "entropy": 0.23398752003908158, + "epoch": 9.43680981595092, + "grad_norm": 1.1164162158966064, + "learning_rate": 3.636992314839606e-06, + "loss": 0.13571034431457518, + "mean_token_accuracy": 0.95239138007164, + "num_tokens": 10190986.0, + "step": 3850 + }, + { + "entropy": 0.2400336007773876, + "epoch": 9.559509202453988, + "grad_norm": 1.1287633180618286, + "learning_rate": 2.2357414248771263e-06, + "loss": 0.13751423835754395, + "mean_token_accuracy": 0.9514365500211716, + "num_tokens": 10318963.0, + "step": 3900 + }, + { + "entropy": 0.22957418337464333, + "epoch": 9.682208588957055, + "grad_norm": 0.8996965885162354, + "learning_rate": 1.1722504278795523e-06, + "loss": 0.13414758682250977, + "mean_token_accuracy": 0.952697988152504, + "num_tokens": 10453673.0, + "step": 3950 + }, + { + "entropy": 0.23466356143355369, + "epoch": 9.804907975460123, + "grad_norm": 1.1095024347305298, + "learning_rate": 4.484651408692574e-07, + "loss": 0.13425475120544433, + "mean_token_accuracy": 0.9517569154500961, + "num_tokens": 10588217.0, + "step": 4000 + }, + { + "entropy": 0.2323933671414852, + "epoch": 9.92760736196319, + "grad_norm": 1.0101295709609985, + "learning_rate": 6.570983808637352e-08, + "loss": 0.13377522468566894, + "mean_token_accuracy": 0.9530322283506394, + "num_tokens": 10725457.0, + "step": 4050 + }, + { + "epoch": 10.0, + "eval_entropy": 0.3864934916155679, + "eval_mean_token_accuracy": 0.7913257755552019, + "eval_not_syn_loss": 0.959854781627655, + "eval_not_syn_runtime": 55.5955, + "eval_not_syn_samples_per_second": 25.092, + "eval_not_syn_steps_per_second": 3.148, + "eval_num_tokens": 10801960.0, + "step": 4080 + }, + { + "epoch": 10.0, + "eval_entropy": 0.36258750983646937, + "eval_mean_token_accuracy": 0.8070946764945984, + "eval_num_tokens": 10801960.0, + "eval_syn_loss": 0.8937228918075562, + "eval_syn_runtime": 58.1703, + "eval_syn_samples_per_second": 23.981, + "eval_syn_steps_per_second": 3.008, + "step": 4080 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.1472674530084928e+17, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +} diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b9596096c71017d49955136a4e23d9f97138d305 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 32, + "lora_bias": false, + "lora_dropout": 0.08602048083239339, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 16, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "k_proj", + "gate_proj", + "up_proj", + "o_proj", + "down_proj", + "v_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..eb526b0ff54644063205e01fa1dc39b7359c3d09 --- /dev/null +++ b/substitutivity_original_Estonian/Qwen3.5-2B-Base_substitutivity_splits_original_features_train_substitutivity_splits_original_features_test1/checkpoint-816/trainer_state.json @@ -0,0 +1,238 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 816, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.2408065378665922, + "epoch": 0.12269938650306748, + "grad_norm": 1.6718658208847046, + "learning_rate": 4.487804970912036e-05, + "loss": 2.129624786376953, + "mean_token_accuracy": 0.5743644836544991, + "num_tokens": 134329.0, + "step": 50 + }, + { + "entropy": 1.1078504550457, + "epoch": 0.24539877300613497, + "grad_norm": 2.4091813564300537, + "learning_rate": 9.067197798373296e-05, + "loss": 1.0346390533447265, + "mean_token_accuracy": 0.7085381114482879, + "num_tokens": 262407.0, + "step": 100 + }, + { + "entropy": 0.9421598988771439, + "epoch": 0.36809815950920244, + "grad_norm": 1.3913646936416626, + "learning_rate": 0.00013646590625834558, + "loss": 0.8716551208496094, + "mean_token_accuracy": 0.7449040985107422, + "num_tokens": 401136.0, + "step": 150 + }, + { + "entropy": 0.8723550146818161, + "epoch": 0.49079754601226994, + "grad_norm": 1.1664468050003052, + "learning_rate": 0.0001822598345329582, + "loss": 0.8095977783203125, + "mean_token_accuracy": 0.7585240352153778, + "num_tokens": 531647.0, + "step": 200 + }, + { + "entropy": 0.8466519421339035, + "epoch": 0.6134969325153374, + "grad_norm": 1.0145421028137207, + "learning_rate": 0.00022805376280757083, + "loss": 0.7830857849121093, + "mean_token_accuracy": 0.7661705195903779, + "num_tokens": 666127.0, + "step": 250 + }, + { + "entropy": 0.8180295366048813, + "epoch": 0.7361963190184049, + "grad_norm": 0.9731705784797668, + "learning_rate": 0.00027384769108218336, + "loss": 0.7573603820800782, + "mean_token_accuracy": 0.7704453605413437, + "num_tokens": 799292.0, + "step": 300 + }, + { + "entropy": 0.7990227991342544, + "epoch": 0.8588957055214724, + "grad_norm": 1.0642374753952026, + "learning_rate": 0.000319641619356796, + "loss": 0.73709228515625, + "mean_token_accuracy": 0.7752733880281448, + "num_tokens": 930614.0, + "step": 350 + }, + { + "entropy": 0.7899087685346603, + "epoch": 0.9815950920245399, + "grad_norm": 0.880613386631012, + "learning_rate": 0.0003654355476314086, + "loss": 0.7347319793701171, + "mean_token_accuracy": 0.7766731631755829, + "num_tokens": 1061654.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.83522472347532, + "eval_mean_token_accuracy": 0.7484098076820374, + "eval_not_syn_loss": 0.7872886061668396, + "eval_not_syn_runtime": 54.8697, + "eval_not_syn_samples_per_second": 25.424, + "eval_not_syn_steps_per_second": 3.189, + "eval_num_tokens": 1080196.0, + "step": 408 + }, + { + "epoch": 1.0, + "eval_entropy": 0.7784442271505083, + "eval_mean_token_accuracy": 0.8021791185651507, + "eval_num_tokens": 1080196.0, + "eval_syn_loss": 0.6980098485946655, + "eval_syn_runtime": 56.6342, + "eval_syn_samples_per_second": 24.632, + "eval_syn_steps_per_second": 3.09, + "step": 408 + }, + { + "entropy": 0.75672573694075, + "epoch": 1.1030674846625768, + "grad_norm": 0.9244198203086853, + "learning_rate": 0.00037356351883456205, + "loss": 0.7048745727539063, + "mean_token_accuracy": 0.783016896609104, + "num_tokens": 1199170.0, + "step": 450 + }, + { + "entropy": 0.7406797379255294, + "epoch": 1.2257668711656442, + "grad_norm": 0.6744896769523621, + "learning_rate": 0.00037311248151765587, + "loss": 0.6860730743408203, + "mean_token_accuracy": 0.787369327545166, + "num_tokens": 1334298.0, + "step": 500 + }, + { + "entropy": 0.7424973237514496, + "epoch": 1.3484662576687116, + "grad_norm": 0.92305588722229, + "learning_rate": 0.000372320629215228, + "loss": 0.6855724334716797, + "mean_token_accuracy": 0.7871134179830551, + "num_tokens": 1463189.0, + "step": 550 + }, + { + "entropy": 0.7502243828773498, + "epoch": 1.471165644171779, + "grad_norm": 0.8259047269821167, + "learning_rate": 0.00037118941074037944, + "loss": 0.6949834442138672, + "mean_token_accuracy": 0.7851914083957672, + "num_tokens": 1590310.0, + "step": 600 + }, + { + "entropy": 0.7233260518312454, + "epoch": 1.5938650306748468, + "grad_norm": 0.882722795009613, + "learning_rate": 0.00036972089582775814, + "loss": 0.6649005889892579, + "mean_token_accuracy": 0.7898879665136337, + "num_tokens": 1721836.0, + "step": 650 + }, + { + "entropy": 0.7162708967924118, + "epoch": 1.716564417177914, + "grad_norm": 0.6593027114868164, + "learning_rate": 0.0003679177713466678, + "loss": 0.6653710174560546, + "mean_token_accuracy": 0.7905678844451904, + "num_tokens": 1853507.0, + "step": 700 + }, + { + "entropy": 0.7150520020723343, + "epoch": 1.8392638036809816, + "grad_norm": 0.665340781211853, + "learning_rate": 0.0003657833363850354, + "loss": 0.6600938415527344, + "mean_token_accuracy": 0.7918649202585221, + "num_tokens": 1988113.0, + "step": 750 + }, + { + "entropy": 0.6993386310338974, + "epoch": 1.961963190184049, + "grad_norm": 0.6502187252044678, + "learning_rate": 0.00036332149621323294, + "loss": 0.6469607543945313, + "mean_token_accuracy": 0.7950778317451477, + "num_tokens": 2121314.0, + "step": 800 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7315139893123082, + "eval_mean_token_accuracy": 0.7713504004478454, + "eval_not_syn_loss": 0.699893593788147, + "eval_not_syn_runtime": 55.4135, + "eval_not_syn_samples_per_second": 25.174, + "eval_not_syn_steps_per_second": 3.158, + "eval_num_tokens": 2160392.0, + "step": 816 + }, + { + "epoch": 2.0, + "eval_entropy": 0.6848451798302787, + "eval_mean_token_accuracy": 0.814895657471248, + "eval_num_tokens": 2160392.0, + "eval_syn_loss": 0.6415576934814453, + "eval_syn_runtime": 56.4735, + "eval_syn_samples_per_second": 24.702, + "eval_syn_steps_per_second": 3.099, + "step": 816 + } + ], + "logging_steps": 50, + "max_steps": 4080, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.308663482309184e+16, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}