diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/README.md new file mode 100644 index 0000000000000000000000000000000000000000..45089d5298dfb507172e185232886d62c3f35162 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/0j86h051) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2a7efe7af0ceadd86d0f851dc87d465191c777 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04180832058159435, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6145e543cda7a2ee7877fd8ceb320723020255f3 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1284/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1284, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8341716039180755, + "epoch": 0.11689070718877849, + "grad_norm": 1.1992322206497192, + "learning_rate": 5.366728168873013e-05, + "loss": 1.6918942260742187, + "mean_token_accuracy": 0.631798365265131, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9513179230690002, + "epoch": 0.23378141437755698, + "grad_norm": 1.1290454864501953, + "learning_rate": 0.00010842981402416902, + "loss": 0.8867578887939453, + "mean_token_accuracy": 0.7587438315153122, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.8510265004634857, + "epoch": 0.3506721215663355, + "grad_norm": 0.7732136845588684, + "learning_rate": 0.00016319234635960792, + "loss": 0.7856448364257812, + "mean_token_accuracy": 0.7769897204637527, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.8142367601394653, + "epoch": 0.46756282875511396, + "grad_norm": 0.7580806016921997, + "learning_rate": 0.00021795487869504684, + "loss": 0.7461666870117187, + "mean_token_accuracy": 0.7888966089487076, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.772578022480011, + "epoch": 0.5844535359438925, + "grad_norm": 0.5330358147621155, + "learning_rate": 0.00027271741103048575, + "loss": 0.7142900848388671, + "mean_token_accuracy": 0.7959607627987861, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7481484657526016, + "epoch": 0.701344243132671, + "grad_norm": 0.8242517709732056, + "learning_rate": 0.00032747994336592464, + "loss": 0.6911422729492187, + "mean_token_accuracy": 0.8028205358982086, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.741372903585434, + "epoch": 0.8182349503214494, + "grad_norm": 0.7722981572151184, + "learning_rate": 0.00038224247570136353, + "loss": 0.6908904266357422, + "mean_token_accuracy": 0.8018302822113037, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.7257227802276611, + "epoch": 0.9351256575102279, + "grad_norm": 0.7969573140144348, + "learning_rate": 0.0004370050080368024, + "loss": 0.6714310455322265, + "mean_token_accuracy": 0.8057891410589219, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8317563653766334, + "eval_loss": 0.8135058283805847, + "eval_mean_token_accuracy": 0.7755894215850087, + "eval_num_tokens": 1325643.0, + "eval_runtime": 37.4951, + "eval_samples_per_second": 32.831, + "eval_steps_per_second": 4.107, + "step": 428 + }, + { + "entropy": 0.7179872094087265, + "epoch": 1.0514319111630626, + "grad_norm": 0.6224768161773682, + "learning_rate": 0.00046873290101955413, + "loss": 0.6683222961425781, + "mean_token_accuracy": 0.8054787133207273, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6991294291615486, + "epoch": 1.1683226183518411, + "grad_norm": 0.6813111901283264, + "learning_rate": 0.00046837443306086947, + "loss": 0.6493977355957031, + "mean_token_accuracy": 0.8089349576830864, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6951853144168854, + "epoch": 1.2852133255406195, + "grad_norm": 0.617091953754425, + "learning_rate": 0.0004676269147738558, + "loss": 0.6484123229980469, + "mean_token_accuracy": 0.8088837671279907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6852494943141937, + "epoch": 1.4021040327293979, + "grad_norm": 0.4677598774433136, + "learning_rate": 0.0004664915890374708, + "loss": 0.6416233062744141, + "mean_token_accuracy": 0.8106312158703805, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6703594943881035, + "epoch": 1.5189947399181765, + "grad_norm": 0.5708025693893433, + "learning_rate": 0.0004649703435278991, + "loss": 0.6273183441162109, + "mean_token_accuracy": 0.8155005398392677, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6676286320388317, + "epoch": 1.635885447106955, + "grad_norm": 0.5555347800254822, + "learning_rate": 0.00046306570757996264, + "loss": 0.6276700973510743, + "mean_token_accuracy": 0.8154828292131424, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6669242936372757, + "epoch": 1.7527761542957334, + "grad_norm": 0.5260112285614014, + "learning_rate": 0.0004607808479816624, + "loss": 0.6228141784667969, + "mean_token_accuracy": 0.8164840793609619, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6598560312390327, + "epoch": 1.869666861484512, + "grad_norm": 0.5677736401557922, + "learning_rate": 0.0004581195637088436, + "loss": 0.6214292907714843, + "mean_token_accuracy": 0.8171795177459716, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6687870016694069, + "epoch": 1.9865575686732906, + "grad_norm": 0.6449615955352783, + "learning_rate": 0.00045508627960873823, + "loss": 0.6243909454345703, + "mean_token_accuracy": 0.8147780740261078, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7786144471013701, + "eval_loss": 0.7599140405654907, + "eval_mean_token_accuracy": 0.7935443106409791, + "eval_num_tokens": 2651286.0, + "eval_runtime": 39.0048, + "eval_samples_per_second": 31.56, + "eval_steps_per_second": 3.948, + "step": 856 + }, + { + "entropy": 0.5821921329701966, + "epoch": 2.102863822326125, + "grad_norm": 0.6116960048675537, + "learning_rate": 0.00045168603904288863, + "loss": 0.5406004714965821, + "mean_token_accuracy": 0.8334921918921734, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5937331764400006, + "epoch": 2.2197545295149035, + "grad_norm": 0.5440122485160828, + "learning_rate": 0.00044792449550168286, + "loss": 0.5515737533569336, + "mean_token_accuracy": 0.8314691257476806, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.5877273553609847, + "epoch": 2.3366452367036823, + "grad_norm": 0.6505260467529297, + "learning_rate": 0.0004438079032044453, + "loss": 0.5507744979858399, + "mean_token_accuracy": 0.8314063146710395, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5897100016474723, + "epoch": 2.4535359438924607, + "grad_norm": 0.54567551612854, + "learning_rate": 0.0004393431067007111, + "loss": 0.5528768157958984, + "mean_token_accuracy": 0.8315245220065117, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5831590622663498, + "epoch": 2.570426651081239, + "grad_norm": 0.44826796650886536, + "learning_rate": 0.00043453752948997376, + "loss": 0.5442767333984375, + "mean_token_accuracy": 0.8327079233527184, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5839906217157841, + "epoch": 2.6873173582700174, + "grad_norm": 0.5067696571350098, + "learning_rate": 0.0004293991616788285, + "loss": 0.5485663604736328, + "mean_token_accuracy": 0.8324281191825866, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5870274990797043, + "epoch": 2.8042080654587958, + "grad_norm": 0.44032806158065796, + "learning_rate": 0.00042393654669603217, + "loss": 0.5524474716186524, + "mean_token_accuracy": 0.8313613015413285, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5864466108381748, + "epoch": 2.9210987726475746, + "grad_norm": 0.5520864129066467, + "learning_rate": 0.00041815876708756964, + "loss": 0.5490786361694336, + "mean_token_accuracy": 0.8314764249324799, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6819627613990338, + "eval_loss": 0.7593190670013428, + "eval_mean_token_accuracy": 0.7857603395914102, + "eval_num_tokens": 3976929.0, + "eval_runtime": 39.3654, + "eval_samples_per_second": 31.271, + "eval_steps_per_second": 3.912, + "step": 1284 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5.959334198185114e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2a7efe7af0ceadd86d0f851dc87d465191c777 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04180832058159435, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a0ea6cfaf3f02d1143a90db14f8085903235a02 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-1712/trainer_state.json @@ -0,0 +1,418 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1712, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8341716039180755, + "epoch": 0.11689070718877849, + "grad_norm": 1.1992322206497192, + "learning_rate": 5.366728168873013e-05, + "loss": 1.6918942260742187, + "mean_token_accuracy": 0.631798365265131, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9513179230690002, + "epoch": 0.23378141437755698, + "grad_norm": 1.1290454864501953, + "learning_rate": 0.00010842981402416902, + "loss": 0.8867578887939453, + "mean_token_accuracy": 0.7587438315153122, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.8510265004634857, + "epoch": 0.3506721215663355, + "grad_norm": 0.7732136845588684, + "learning_rate": 0.00016319234635960792, + "loss": 0.7856448364257812, + "mean_token_accuracy": 0.7769897204637527, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.8142367601394653, + "epoch": 0.46756282875511396, + "grad_norm": 0.7580806016921997, + "learning_rate": 0.00021795487869504684, + "loss": 0.7461666870117187, + "mean_token_accuracy": 0.7888966089487076, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.772578022480011, + "epoch": 0.5844535359438925, + "grad_norm": 0.5330358147621155, + "learning_rate": 0.00027271741103048575, + "loss": 0.7142900848388671, + "mean_token_accuracy": 0.7959607627987861, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7481484657526016, + "epoch": 0.701344243132671, + "grad_norm": 0.8242517709732056, + "learning_rate": 0.00032747994336592464, + "loss": 0.6911422729492187, + "mean_token_accuracy": 0.8028205358982086, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.741372903585434, + "epoch": 0.8182349503214494, + "grad_norm": 0.7722981572151184, + "learning_rate": 0.00038224247570136353, + "loss": 0.6908904266357422, + "mean_token_accuracy": 0.8018302822113037, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.7257227802276611, + "epoch": 0.9351256575102279, + "grad_norm": 0.7969573140144348, + "learning_rate": 0.0004370050080368024, + "loss": 0.6714310455322265, + "mean_token_accuracy": 0.8057891410589219, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8317563653766334, + "eval_loss": 0.8135058283805847, + "eval_mean_token_accuracy": 0.7755894215850087, + "eval_num_tokens": 1325643.0, + "eval_runtime": 37.4951, + "eval_samples_per_second": 32.831, + "eval_steps_per_second": 4.107, + "step": 428 + }, + { + "entropy": 0.7179872094087265, + "epoch": 1.0514319111630626, + "grad_norm": 0.6224768161773682, + "learning_rate": 0.00046873290101955413, + "loss": 0.6683222961425781, + "mean_token_accuracy": 0.8054787133207273, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6991294291615486, + "epoch": 1.1683226183518411, + "grad_norm": 0.6813111901283264, + "learning_rate": 0.00046837443306086947, + "loss": 0.6493977355957031, + "mean_token_accuracy": 0.8089349576830864, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6951853144168854, + "epoch": 1.2852133255406195, + "grad_norm": 0.617091953754425, + "learning_rate": 0.0004676269147738558, + "loss": 0.6484123229980469, + "mean_token_accuracy": 0.8088837671279907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6852494943141937, + "epoch": 1.4021040327293979, + "grad_norm": 0.4677598774433136, + "learning_rate": 0.0004664915890374708, + "loss": 0.6416233062744141, + "mean_token_accuracy": 0.8106312158703805, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6703594943881035, + "epoch": 1.5189947399181765, + "grad_norm": 0.5708025693893433, + "learning_rate": 0.0004649703435278991, + "loss": 0.6273183441162109, + "mean_token_accuracy": 0.8155005398392677, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6676286320388317, + "epoch": 1.635885447106955, + "grad_norm": 0.5555347800254822, + "learning_rate": 0.00046306570757996264, + "loss": 0.6276700973510743, + "mean_token_accuracy": 0.8154828292131424, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6669242936372757, + "epoch": 1.7527761542957334, + "grad_norm": 0.5260112285614014, + "learning_rate": 0.0004607808479816624, + "loss": 0.6228141784667969, + "mean_token_accuracy": 0.8164840793609619, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6598560312390327, + "epoch": 1.869666861484512, + "grad_norm": 0.5677736401557922, + "learning_rate": 0.0004581195637088436, + "loss": 0.6214292907714843, + "mean_token_accuracy": 0.8171795177459716, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6687870016694069, + "epoch": 1.9865575686732906, + "grad_norm": 0.6449615955352783, + "learning_rate": 0.00045508627960873823, + "loss": 0.6243909454345703, + "mean_token_accuracy": 0.8147780740261078, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7786144471013701, + "eval_loss": 0.7599140405654907, + "eval_mean_token_accuracy": 0.7935443106409791, + "eval_num_tokens": 2651286.0, + "eval_runtime": 39.0048, + "eval_samples_per_second": 31.56, + "eval_steps_per_second": 3.948, + "step": 856 + }, + { + "entropy": 0.5821921329701966, + "epoch": 2.102863822326125, + "grad_norm": 0.6116960048675537, + "learning_rate": 0.00045168603904288863, + "loss": 0.5406004714965821, + "mean_token_accuracy": 0.8334921918921734, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5937331764400006, + "epoch": 2.2197545295149035, + "grad_norm": 0.5440122485160828, + "learning_rate": 0.00044792449550168286, + "loss": 0.5515737533569336, + "mean_token_accuracy": 0.8314691257476806, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.5877273553609847, + "epoch": 2.3366452367036823, + "grad_norm": 0.6505260467529297, + "learning_rate": 0.0004438079032044453, + "loss": 0.5507744979858399, + "mean_token_accuracy": 0.8314063146710395, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5897100016474723, + "epoch": 2.4535359438924607, + "grad_norm": 0.54567551612854, + "learning_rate": 0.0004393431067007111, + "loss": 0.5528768157958984, + "mean_token_accuracy": 0.8315245220065117, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5831590622663498, + "epoch": 2.570426651081239, + "grad_norm": 0.44826796650886536, + "learning_rate": 0.00043453752948997376, + "loss": 0.5442767333984375, + "mean_token_accuracy": 0.8327079233527184, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5839906217157841, + "epoch": 2.6873173582700174, + "grad_norm": 0.5067696571350098, + "learning_rate": 0.0004293991616788285, + "loss": 0.5485663604736328, + "mean_token_accuracy": 0.8324281191825866, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5870274990797043, + "epoch": 2.8042080654587958, + "grad_norm": 0.44032806158065796, + "learning_rate": 0.00042393654669603217, + "loss": 0.5524474716186524, + "mean_token_accuracy": 0.8313613015413285, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5864466108381748, + "epoch": 2.9210987726475746, + "grad_norm": 0.5520864129066467, + "learning_rate": 0.00041815876708756964, + "loss": 0.5490786361694336, + "mean_token_accuracy": 0.8314764249324799, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6819627613990338, + "eval_loss": 0.7593190670013428, + "eval_mean_token_accuracy": 0.7857603395914102, + "eval_num_tokens": 3976929.0, + "eval_runtime": 39.3654, + "eval_samples_per_second": 31.271, + "eval_steps_per_second": 3.912, + "step": 1284 + }, + { + "entropy": 0.5587641523411525, + "epoch": 3.037405026300409, + "grad_norm": 0.622401773929596, + "learning_rate": 0.0004120754294153441, + "loss": 0.518932762145996, + "mean_token_accuracy": 0.8399260457436643, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.5028628017008304, + "epoch": 3.1542957334891875, + "grad_norm": 0.577396810054779, + "learning_rate": 0.00040569664828459917, + "loss": 0.4589382171630859, + "mean_token_accuracy": 0.8538418188691139, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.5227407096326351, + "epoch": 3.2711864406779663, + "grad_norm": 0.5113071203231812, + "learning_rate": 0.00039903302952663176, + "loss": 0.47863777160644533, + "mean_token_accuracy": 0.847364938557148, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.5050077450275421, + "epoch": 3.3880771478667446, + "grad_norm": 0.48977184295654297, + "learning_rate": 0.0003920956525647558, + "loss": 0.46841480255126955, + "mean_token_accuracy": 0.8519425508379936, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.5163291451334954, + "epoch": 3.504967855055523, + "grad_norm": 0.509864330291748, + "learning_rate": 0.000384896051992837, + "loss": 0.47587432861328127, + "mean_token_accuracy": 0.8486990982294083, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.5134807989001274, + "epoch": 3.6218585622443014, + "grad_norm": 0.46544620394706726, + "learning_rate": 0.00037744619839702735, + "loss": 0.47692710876464844, + "mean_token_accuracy": 0.8489899519085884, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.5216251534223556, + "epoch": 3.73874926943308, + "grad_norm": 0.5167004466056824, + "learning_rate": 0.0003697584784525874, + "loss": 0.4837848663330078, + "mean_token_accuracy": 0.847066233754158, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.5254101701080799, + "epoch": 3.8556399766218585, + "grad_norm": 0.5504006743431091, + "learning_rate": 0.00036184567432888745, + "loss": 0.48759506225585936, + "mean_token_accuracy": 0.8457375919818878, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.5107753933966159, + "epoch": 3.972530683810637, + "grad_norm": 0.421975314617157, + "learning_rate": 0.0003537209424368311, + "loss": 0.4743759536743164, + "mean_token_accuracy": 0.8505110186338425, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.665543915002377, + "eval_loss": 0.7487082481384277, + "eval_mean_token_accuracy": 0.7999678904359991, + "eval_num_tokens": 5302572.0, + "eval_runtime": 38.5439, + "eval_samples_per_second": 31.938, + "eval_steps_per_second": 3.995, + "step": 1712 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7.944979792148429e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2a7efe7af0ceadd86d0f851dc87d465191c777 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04180832058159435, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4a82769919aa39d3d91dee8e6dc306f0b2c9c039 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2140/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8341716039180755, + "epoch": 0.11689070718877849, + "grad_norm": 1.1992322206497192, + "learning_rate": 5.366728168873013e-05, + "loss": 1.6918942260742187, + "mean_token_accuracy": 0.631798365265131, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9513179230690002, + "epoch": 0.23378141437755698, + "grad_norm": 1.1290454864501953, + "learning_rate": 0.00010842981402416902, + "loss": 0.8867578887939453, + "mean_token_accuracy": 0.7587438315153122, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.8510265004634857, + "epoch": 0.3506721215663355, + "grad_norm": 0.7732136845588684, + "learning_rate": 0.00016319234635960792, + "loss": 0.7856448364257812, + "mean_token_accuracy": 0.7769897204637527, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.8142367601394653, + "epoch": 0.46756282875511396, + "grad_norm": 0.7580806016921997, + "learning_rate": 0.00021795487869504684, + "loss": 0.7461666870117187, + "mean_token_accuracy": 0.7888966089487076, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.772578022480011, + "epoch": 0.5844535359438925, + "grad_norm": 0.5330358147621155, + "learning_rate": 0.00027271741103048575, + "loss": 0.7142900848388671, + "mean_token_accuracy": 0.7959607627987861, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7481484657526016, + "epoch": 0.701344243132671, + "grad_norm": 0.8242517709732056, + "learning_rate": 0.00032747994336592464, + "loss": 0.6911422729492187, + "mean_token_accuracy": 0.8028205358982086, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.741372903585434, + "epoch": 0.8182349503214494, + "grad_norm": 0.7722981572151184, + "learning_rate": 0.00038224247570136353, + "loss": 0.6908904266357422, + "mean_token_accuracy": 0.8018302822113037, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.7257227802276611, + "epoch": 0.9351256575102279, + "grad_norm": 0.7969573140144348, + "learning_rate": 0.0004370050080368024, + "loss": 0.6714310455322265, + "mean_token_accuracy": 0.8057891410589219, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.8317563653766334, + "eval_loss": 0.8135058283805847, + "eval_mean_token_accuracy": 0.7755894215850087, + "eval_num_tokens": 1325643.0, + "eval_runtime": 37.4951, + "eval_samples_per_second": 32.831, + "eval_steps_per_second": 4.107, + "step": 428 + }, + { + "entropy": 0.7179872094087265, + "epoch": 1.0514319111630626, + "grad_norm": 0.6224768161773682, + "learning_rate": 0.00046873290101955413, + "loss": 0.6683222961425781, + "mean_token_accuracy": 0.8054787133207273, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6991294291615486, + "epoch": 1.1683226183518411, + "grad_norm": 0.6813111901283264, + "learning_rate": 0.00046837443306086947, + "loss": 0.6493977355957031, + "mean_token_accuracy": 0.8089349576830864, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6951853144168854, + "epoch": 1.2852133255406195, + "grad_norm": 0.617091953754425, + "learning_rate": 0.0004676269147738558, + "loss": 0.6484123229980469, + "mean_token_accuracy": 0.8088837671279907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6852494943141937, + "epoch": 1.4021040327293979, + "grad_norm": 0.4677598774433136, + "learning_rate": 0.0004664915890374708, + "loss": 0.6416233062744141, + "mean_token_accuracy": 0.8106312158703805, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6703594943881035, + "epoch": 1.5189947399181765, + "grad_norm": 0.5708025693893433, + "learning_rate": 0.0004649703435278991, + "loss": 0.6273183441162109, + "mean_token_accuracy": 0.8155005398392677, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6676286320388317, + "epoch": 1.635885447106955, + "grad_norm": 0.5555347800254822, + "learning_rate": 0.00046306570757996264, + "loss": 0.6276700973510743, + "mean_token_accuracy": 0.8154828292131424, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6669242936372757, + "epoch": 1.7527761542957334, + "grad_norm": 0.5260112285614014, + "learning_rate": 0.0004607808479816624, + "loss": 0.6228141784667969, + "mean_token_accuracy": 0.8164840793609619, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6598560312390327, + "epoch": 1.869666861484512, + "grad_norm": 0.5677736401557922, + "learning_rate": 0.0004581195637088436, + "loss": 0.6214292907714843, + "mean_token_accuracy": 0.8171795177459716, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6687870016694069, + "epoch": 1.9865575686732906, + "grad_norm": 0.6449615955352783, + "learning_rate": 0.00045508627960873823, + "loss": 0.6243909454345703, + "mean_token_accuracy": 0.8147780740261078, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.7786144471013701, + "eval_loss": 0.7599140405654907, + "eval_mean_token_accuracy": 0.7935443106409791, + "eval_num_tokens": 2651286.0, + "eval_runtime": 39.0048, + "eval_samples_per_second": 31.56, + "eval_steps_per_second": 3.948, + "step": 856 + }, + { + "entropy": 0.5821921329701966, + "epoch": 2.102863822326125, + "grad_norm": 0.6116960048675537, + "learning_rate": 0.00045168603904288863, + "loss": 0.5406004714965821, + "mean_token_accuracy": 0.8334921918921734, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5937331764400006, + "epoch": 2.2197545295149035, + "grad_norm": 0.5440122485160828, + "learning_rate": 0.00044792449550168286, + "loss": 0.5515737533569336, + "mean_token_accuracy": 0.8314691257476806, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.5877273553609847, + "epoch": 2.3366452367036823, + "grad_norm": 0.6505260467529297, + "learning_rate": 0.0004438079032044453, + "loss": 0.5507744979858399, + "mean_token_accuracy": 0.8314063146710395, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5897100016474723, + "epoch": 2.4535359438924607, + "grad_norm": 0.54567551612854, + "learning_rate": 0.0004393431067007111, + "loss": 0.5528768157958984, + "mean_token_accuracy": 0.8315245220065117, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5831590622663498, + "epoch": 2.570426651081239, + "grad_norm": 0.44826796650886536, + "learning_rate": 0.00043453752948997376, + "loss": 0.5442767333984375, + "mean_token_accuracy": 0.8327079233527184, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5839906217157841, + "epoch": 2.6873173582700174, + "grad_norm": 0.5067696571350098, + "learning_rate": 0.0004293991616788285, + "loss": 0.5485663604736328, + "mean_token_accuracy": 0.8324281191825866, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5870274990797043, + "epoch": 2.8042080654587958, + "grad_norm": 0.44032806158065796, + "learning_rate": 0.00042393654669603217, + "loss": 0.5524474716186524, + "mean_token_accuracy": 0.8313613015413285, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5864466108381748, + "epoch": 2.9210987726475746, + "grad_norm": 0.5520864129066467, + "learning_rate": 0.00041815876708756964, + "loss": 0.5490786361694336, + "mean_token_accuracy": 0.8314764249324799, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6819627613990338, + "eval_loss": 0.7593190670013428, + "eval_mean_token_accuracy": 0.7857603395914102, + "eval_num_tokens": 3976929.0, + "eval_runtime": 39.3654, + "eval_samples_per_second": 31.271, + "eval_steps_per_second": 3.912, + "step": 1284 + }, + { + "entropy": 0.5587641523411525, + "epoch": 3.037405026300409, + "grad_norm": 0.622401773929596, + "learning_rate": 0.0004120754294153441, + "loss": 0.518932762145996, + "mean_token_accuracy": 0.8399260457436643, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.5028628017008304, + "epoch": 3.1542957334891875, + "grad_norm": 0.577396810054779, + "learning_rate": 0.00040569664828459917, + "loss": 0.4589382171630859, + "mean_token_accuracy": 0.8538418188691139, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.5227407096326351, + "epoch": 3.2711864406779663, + "grad_norm": 0.5113071203231812, + "learning_rate": 0.00039903302952663176, + "loss": 0.47863777160644533, + "mean_token_accuracy": 0.847364938557148, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.5050077450275421, + "epoch": 3.3880771478667446, + "grad_norm": 0.48977184295654297, + "learning_rate": 0.0003920956525647558, + "loss": 0.46841480255126955, + "mean_token_accuracy": 0.8519425508379936, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.5163291451334954, + "epoch": 3.504967855055523, + "grad_norm": 0.509864330291748, + "learning_rate": 0.000384896051992837, + "loss": 0.47587432861328127, + "mean_token_accuracy": 0.8486990982294083, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.5134807989001274, + "epoch": 3.6218585622443014, + "grad_norm": 0.46544620394706726, + "learning_rate": 0.00037744619839702735, + "loss": 0.47692710876464844, + "mean_token_accuracy": 0.8489899519085884, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.5216251534223556, + "epoch": 3.73874926943308, + "grad_norm": 0.5167004466056824, + "learning_rate": 0.0003697584784525874, + "loss": 0.4837848663330078, + "mean_token_accuracy": 0.847066233754158, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.5254101701080799, + "epoch": 3.8556399766218585, + "grad_norm": 0.5504006743431091, + "learning_rate": 0.00036184567432888745, + "loss": 0.48759506225585936, + "mean_token_accuracy": 0.8457375919818878, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.5107753933966159, + "epoch": 3.972530683810637, + "grad_norm": 0.421975314617157, + "learning_rate": 0.0003537209424368311, + "loss": 0.4743759536743164, + "mean_token_accuracy": 0.8505110186338425, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.665543915002377, + "eval_loss": 0.7487082481384277, + "eval_mean_token_accuracy": 0.7999678904359991, + "eval_num_tokens": 5302572.0, + "eval_runtime": 38.5439, + "eval_samples_per_second": 31.938, + "eval_steps_per_second": 3.995, + "step": 1712 + }, + { + "entropy": 0.4414465431891494, + "epoch": 4.088836937463472, + "grad_norm": 0.5308493971824646, + "learning_rate": 0.0003453977915540383, + "loss": 0.39650299072265627, + "mean_token_accuracy": 0.8703725772287378, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.42591195791959763, + "epoch": 4.20572764465225, + "grad_norm": 0.6111563444137573, + "learning_rate": 0.00033689006036415585, + "loss": 0.37838283538818357, + "mean_token_accuracy": 0.874235480427742, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.44086351931095125, + "epoch": 4.322618351841029, + "grad_norm": 0.5524880290031433, + "learning_rate": 0.0003282118944476435, + "loss": 0.3920352554321289, + "mean_token_accuracy": 0.8694414687156677, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.4380896310508251, + "epoch": 4.439509059029807, + "grad_norm": 0.5023863315582275, + "learning_rate": 0.0003193777227622898, + "loss": 0.3916081237792969, + "mean_token_accuracy": 0.8697139009833336, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.44005885019898416, + "epoch": 4.556399766218585, + "grad_norm": 0.455997109413147, + "learning_rate": 0.000310402233652564, + "loss": 0.3955466842651367, + "mean_token_accuracy": 0.8693471103906631, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.444269048422575, + "epoch": 4.673290473407365, + "grad_norm": 0.4542011320590973, + "learning_rate": 0.00030130035042769316, + "loss": 0.3988466262817383, + "mean_token_accuracy": 0.8680117425322532, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.43520899042487143, + "epoch": 4.790181180596143, + "grad_norm": 0.49051445722579956, + "learning_rate": 0.0002920872065490688, + "loss": 0.39819797515869143, + "mean_token_accuracy": 0.8678940117359162, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.429456724524498, + "epoch": 4.907071887784921, + "grad_norm": 0.5267980098724365, + "learning_rate": 0.0002827781204682396, + "loss": 0.39413917541503907, + "mean_token_accuracy": 0.870051506459713, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.5826305385146823, + "eval_loss": 0.7694042921066284, + "eval_mean_token_accuracy": 0.7950637824349589, + "eval_num_tokens": 6628215.0, + "eval_runtime": 38.5641, + "eval_samples_per_second": 31.921, + "eval_steps_per_second": 3.993, + "step": 2140 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9.930445154815795e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2a7efe7af0ceadd86d0f851dc87d465191c777 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test1/checkpoint-2568/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.04180832058159435, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/README.md new file mode 100644 index 0000000000000000000000000000000000000000..0a5b2b9ee63291ee8b2413002adf5e1b1c3636bd --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/README.md @@ -0,0 +1,58 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: transformers +model_name: Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2 +tags: +- generated_from_trainer +- sft +- trl +licence: license +--- + +# Model Card for Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2 + +This model is a fine-tuned version of [Qwen/Qwen3.5-2B-Base](https://huggingface.co/Qwen/Qwen3.5-2B-Base). +It has been trained using [TRL](https://github.com/huggingface/trl). + +## Quick start + +```python +from transformers import pipeline + +question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?" +generator = pipeline("text-generation", model="None", device="cuda") +output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0] +print(output["generated_text"]) +``` + +## Training procedure + +[Visualize in Weights & Biases](https://wandb.ai/katriin-kukk/Cross_lingual_morphological_generalization/runs/6q4yr6mt) + + + +This model was trained with SFT. + +### Framework versions + +- TRL: 0.29.0 +- Transformers: 5.5.4 +- Pytorch: 2.10.0 +- Datasets: 4.6.1 +- Tokenizers: 0.22.2 + +## Citations + + + +Cite TRL as: + +```bibtex +@software{vonwerra2020trl, + title = {{TRL: Transformers Reinforcement Learning}}, + author = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin}, + license = {Apache-2.0}, + url = {https://github.com/huggingface/trl}, + year = {2020} +} +``` \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6afdf7c56498968853e88aaa6acf5b7760df48ab --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1284/trainer_state.json @@ -0,0 +1,317 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1284, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6.142907198829312e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bd1cdcabe854134d12b7611ddff01efda9f73c12 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-1712/trainer_state.json @@ -0,0 +1,418 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 4.0, + "eval_steps": 500, + "global_step": 1712, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8.189719176112896e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6928b20771048bd0277ae0671aca8b8ae240f925 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2140/trainer_state.json @@ -0,0 +1,509 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 5.0, + "eval_steps": 500, + "global_step": 2140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.0236345370205184e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..50bb6eedd8378dfc2a02b04a5e958e3882db7456 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2568/trainer_state.json @@ -0,0 +1,610 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 6.0, + "eval_steps": 500, + "global_step": 2568, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + }, + { + "entropy": 0.38985791001188097, + "epoch": 5.023378141437756, + "grad_norm": 0.7454882860183716, + "learning_rate": 0.00015375688837676246, + "loss": 0.32916854858398437, + "mean_token_accuracy": 0.8886270373310875, + "num_tokens": 6662015.0, + "step": 2150 + }, + { + "entropy": 0.30170531339943407, + "epoch": 5.140268848626534, + "grad_norm": 0.5329811573028564, + "learning_rate": 0.0001484396230918344, + "loss": 0.23112209320068358, + "mean_token_accuracy": 0.9198204749822616, + "num_tokens": 6817989.0, + "step": 2200 + }, + { + "entropy": 0.3075008810311556, + "epoch": 5.257159555815313, + "grad_norm": 0.6896093487739563, + "learning_rate": 0.00014309472490462715, + "loss": 0.2416652488708496, + "mean_token_accuracy": 0.9158366405963898, + "num_tokens": 6977021.0, + "step": 2250 + }, + { + "entropy": 0.31153449676930905, + "epoch": 5.374050263004091, + "grad_norm": 0.6930833458900452, + "learning_rate": 0.00013773108063651161, + "loss": 0.24753889083862304, + "mean_token_accuracy": 0.9136614826321602, + "num_tokens": 7133360.0, + "step": 2300 + }, + { + "entropy": 0.3154346436262131, + "epoch": 5.490940970192869, + "grad_norm": 0.6476582884788513, + "learning_rate": 0.0001323576082774734, + "loss": 0.24872392654418946, + "mean_token_accuracy": 0.9135914054512978, + "num_tokens": 7282372.0, + "step": 2350 + }, + { + "entropy": 0.313641587048769, + "epoch": 5.607831677381649, + "grad_norm": 0.6953482627868652, + "learning_rate": 0.0001269832421584059, + "loss": 0.24919464111328124, + "mean_token_accuracy": 0.9131662029027939, + "num_tokens": 7435766.0, + "step": 2400 + }, + { + "entropy": 0.3163894522190094, + "epoch": 5.724722384570427, + "grad_norm": 0.6673940420150757, + "learning_rate": 0.00012161691809623384, + "loss": 0.25111726760864256, + "mean_token_accuracy": 0.913602537214756, + "num_tokens": 7578408.0, + "step": 2450 + }, + { + "entropy": 0.3183368914574385, + "epoch": 5.841613091759205, + "grad_norm": 0.7174487709999084, + "learning_rate": 0.00011626755853656576, + "loss": 0.2518378448486328, + "mean_token_accuracy": 0.912000589966774, + "num_tokens": 7734599.0, + "step": 2500 + }, + { + "entropy": 0.3077564483135939, + "epoch": 5.958503798947984, + "grad_norm": 0.6662200689315796, + "learning_rate": 0.00011094405771857891, + "loss": 0.24354087829589843, + "mean_token_accuracy": 0.9140791621804237, + "num_tokens": 7898028.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5403795917699863, + "eval_loss": 0.8572548627853394, + "eval_mean_token_accuracy": 0.796143418395674, + "eval_num_tokens": 7953858.0, + "eval_runtime": 36.7176, + "eval_samples_per_second": 33.526, + "eval_steps_per_second": 4.194, + "step": 2568 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.228067205913728e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..0538ba571eaece25d59271908a07300dca6ecb5c --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-2996/trainer_state.json @@ -0,0 +1,701 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 7.0, + "eval_steps": 500, + "global_step": 2996, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + }, + { + "entropy": 0.38985791001188097, + "epoch": 5.023378141437756, + "grad_norm": 0.7454882860183716, + "learning_rate": 0.00015375688837676246, + "loss": 0.32916854858398437, + "mean_token_accuracy": 0.8886270373310875, + "num_tokens": 6662015.0, + "step": 2150 + }, + { + "entropy": 0.30170531339943407, + "epoch": 5.140268848626534, + "grad_norm": 0.5329811573028564, + "learning_rate": 0.0001484396230918344, + "loss": 0.23112209320068358, + "mean_token_accuracy": 0.9198204749822616, + "num_tokens": 6817989.0, + "step": 2200 + }, + { + "entropy": 0.3075008810311556, + "epoch": 5.257159555815313, + "grad_norm": 0.6896093487739563, + "learning_rate": 0.00014309472490462715, + "loss": 0.2416652488708496, + "mean_token_accuracy": 0.9158366405963898, + "num_tokens": 6977021.0, + "step": 2250 + }, + { + "entropy": 0.31153449676930905, + "epoch": 5.374050263004091, + "grad_norm": 0.6930833458900452, + "learning_rate": 0.00013773108063651161, + "loss": 0.24753889083862304, + "mean_token_accuracy": 0.9136614826321602, + "num_tokens": 7133360.0, + "step": 2300 + }, + { + "entropy": 0.3154346436262131, + "epoch": 5.490940970192869, + "grad_norm": 0.6476582884788513, + "learning_rate": 0.0001323576082774734, + "loss": 0.24872392654418946, + "mean_token_accuracy": 0.9135914054512978, + "num_tokens": 7282372.0, + "step": 2350 + }, + { + "entropy": 0.313641587048769, + "epoch": 5.607831677381649, + "grad_norm": 0.6953482627868652, + "learning_rate": 0.0001269832421584059, + "loss": 0.24919464111328124, + "mean_token_accuracy": 0.9131662029027939, + "num_tokens": 7435766.0, + "step": 2400 + }, + { + "entropy": 0.3163894522190094, + "epoch": 5.724722384570427, + "grad_norm": 0.6673940420150757, + "learning_rate": 0.00012161691809623384, + "loss": 0.25111726760864256, + "mean_token_accuracy": 0.913602537214756, + "num_tokens": 7578408.0, + "step": 2450 + }, + { + "entropy": 0.3183368914574385, + "epoch": 5.841613091759205, + "grad_norm": 0.7174487709999084, + "learning_rate": 0.00011626755853656576, + "loss": 0.2518378448486328, + "mean_token_accuracy": 0.912000589966774, + "num_tokens": 7734599.0, + "step": 2500 + }, + { + "entropy": 0.3077564483135939, + "epoch": 5.958503798947984, + "grad_norm": 0.6662200689315796, + "learning_rate": 0.00011094405771857891, + "loss": 0.24354087829589843, + "mean_token_accuracy": 0.9140791621804237, + "num_tokens": 7898028.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5403795917699863, + "eval_loss": 0.8572548627853394, + "eval_mean_token_accuracy": 0.796143418395674, + "eval_num_tokens": 7953858.0, + "eval_runtime": 36.7176, + "eval_samples_per_second": 33.526, + "eval_steps_per_second": 4.194, + "step": 2568 + }, + { + "entropy": 0.25110363144071857, + "epoch": 6.074810052600818, + "grad_norm": 0.6466763019561768, + "learning_rate": 0.0001056552668868023, + "loss": 0.1814822769165039, + "mean_token_accuracy": 0.9370300626035911, + "num_tokens": 8052646.0, + "step": 2600 + }, + { + "entropy": 0.2063255251944065, + "epoch": 6.1917007597895966, + "grad_norm": 0.6950973272323608, + "learning_rate": 0.0001004099795743853, + "loss": 0.13960565567016603, + "mean_token_accuracy": 0.9511301657557487, + "num_tokens": 8212323.0, + "step": 2650 + }, + { + "entropy": 0.20775293290615082, + "epoch": 6.308591466978375, + "grad_norm": 0.660478413105011, + "learning_rate": 9.52169169823223e-05, + "loss": 0.14319146156311036, + "mean_token_accuracy": 0.9500676044821739, + "num_tokens": 8366715.0, + "step": 2700 + }, + { + "entropy": 0.20977566957473756, + "epoch": 6.425482174167154, + "grad_norm": 0.6141477823257446, + "learning_rate": 9.00847134789411e-05, + "loss": 0.14525352478027342, + "mean_token_accuracy": 0.9490846493840217, + "num_tokens": 8515955.0, + "step": 2750 + }, + { + "entropy": 0.2143413795530796, + "epoch": 6.5423728813559325, + "grad_norm": 0.612184464931488, + "learning_rate": 8.502190224376661e-05, + "loss": 0.14751774787902833, + "mean_token_accuracy": 0.9478235357999801, + "num_tokens": 8672361.0, + "step": 2800 + }, + { + "entropy": 0.21027401253581046, + "epoch": 6.659263588544711, + "grad_norm": 0.644019365310669, + "learning_rate": 8.003690107962787e-05, + "loss": 0.1495517921447754, + "mean_token_accuracy": 0.9470956793427467, + "num_tokens": 8827819.0, + "step": 2850 + }, + { + "entropy": 0.2109992092102766, + "epoch": 6.776154295733489, + "grad_norm": 0.5456500053405762, + "learning_rate": 7.513799841659927e-05, + "loss": 0.14893126487731934, + "mean_token_accuracy": 0.9479587042331695, + "num_tokens": 8979535.0, + "step": 2900 + }, + { + "entropy": 0.20949276342988013, + "epoch": 6.893045002922268, + "grad_norm": 0.6803008913993835, + "learning_rate": 7.033333953104615e-05, + "loss": 0.1461741542816162, + "mean_token_accuracy": 0.9480843073129654, + "num_tokens": 9134134.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.4759114793949313, + "eval_loss": 0.9827883839607239, + "eval_mean_token_accuracy": 0.7945680889216337, + "eval_num_tokens": 9279501.0, + "eval_runtime": 36.4188, + "eval_samples_per_second": 33.801, + "eval_steps_per_second": 4.229, + "step": 2996 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.4340983113207296e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6f3bf7a177ee42ac40a722b38f4f97628a909364 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3424/trainer_state.json @@ -0,0 +1,802 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 8.0, + "eval_steps": 500, + "global_step": 3424, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + }, + { + "entropy": 0.38985791001188097, + "epoch": 5.023378141437756, + "grad_norm": 0.7454882860183716, + "learning_rate": 0.00015375688837676246, + "loss": 0.32916854858398437, + "mean_token_accuracy": 0.8886270373310875, + "num_tokens": 6662015.0, + "step": 2150 + }, + { + "entropy": 0.30170531339943407, + "epoch": 5.140268848626534, + "grad_norm": 0.5329811573028564, + "learning_rate": 0.0001484396230918344, + "loss": 0.23112209320068358, + "mean_token_accuracy": 0.9198204749822616, + "num_tokens": 6817989.0, + "step": 2200 + }, + { + "entropy": 0.3075008810311556, + "epoch": 5.257159555815313, + "grad_norm": 0.6896093487739563, + "learning_rate": 0.00014309472490462715, + "loss": 0.2416652488708496, + "mean_token_accuracy": 0.9158366405963898, + "num_tokens": 6977021.0, + "step": 2250 + }, + { + "entropy": 0.31153449676930905, + "epoch": 5.374050263004091, + "grad_norm": 0.6930833458900452, + "learning_rate": 0.00013773108063651161, + "loss": 0.24753889083862304, + "mean_token_accuracy": 0.9136614826321602, + "num_tokens": 7133360.0, + "step": 2300 + }, + { + "entropy": 0.3154346436262131, + "epoch": 5.490940970192869, + "grad_norm": 0.6476582884788513, + "learning_rate": 0.0001323576082774734, + "loss": 0.24872392654418946, + "mean_token_accuracy": 0.9135914054512978, + "num_tokens": 7282372.0, + "step": 2350 + }, + { + "entropy": 0.313641587048769, + "epoch": 5.607831677381649, + "grad_norm": 0.6953482627868652, + "learning_rate": 0.0001269832421584059, + "loss": 0.24919464111328124, + "mean_token_accuracy": 0.9131662029027939, + "num_tokens": 7435766.0, + "step": 2400 + }, + { + "entropy": 0.3163894522190094, + "epoch": 5.724722384570427, + "grad_norm": 0.6673940420150757, + "learning_rate": 0.00012161691809623384, + "loss": 0.25111726760864256, + "mean_token_accuracy": 0.913602537214756, + "num_tokens": 7578408.0, + "step": 2450 + }, + { + "entropy": 0.3183368914574385, + "epoch": 5.841613091759205, + "grad_norm": 0.7174487709999084, + "learning_rate": 0.00011626755853656576, + "loss": 0.2518378448486328, + "mean_token_accuracy": 0.912000589966774, + "num_tokens": 7734599.0, + "step": 2500 + }, + { + "entropy": 0.3077564483135939, + "epoch": 5.958503798947984, + "grad_norm": 0.6662200689315796, + "learning_rate": 0.00011094405771857891, + "loss": 0.24354087829589843, + "mean_token_accuracy": 0.9140791621804237, + "num_tokens": 7898028.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5403795917699863, + "eval_loss": 0.8572548627853394, + "eval_mean_token_accuracy": 0.796143418395674, + "eval_num_tokens": 7953858.0, + "eval_runtime": 36.7176, + "eval_samples_per_second": 33.526, + "eval_steps_per_second": 4.194, + "step": 2568 + }, + { + "entropy": 0.25110363144071857, + "epoch": 6.074810052600818, + "grad_norm": 0.6466763019561768, + "learning_rate": 0.0001056552668868023, + "loss": 0.1814822769165039, + "mean_token_accuracy": 0.9370300626035911, + "num_tokens": 8052646.0, + "step": 2600 + }, + { + "entropy": 0.2063255251944065, + "epoch": 6.1917007597895966, + "grad_norm": 0.6950973272323608, + "learning_rate": 0.0001004099795743853, + "loss": 0.13960565567016603, + "mean_token_accuracy": 0.9511301657557487, + "num_tokens": 8212323.0, + "step": 2650 + }, + { + "entropy": 0.20775293290615082, + "epoch": 6.308591466978375, + "grad_norm": 0.660478413105011, + "learning_rate": 9.52169169823223e-05, + "loss": 0.14319146156311036, + "mean_token_accuracy": 0.9500676044821739, + "num_tokens": 8366715.0, + "step": 2700 + }, + { + "entropy": 0.20977566957473756, + "epoch": 6.425482174167154, + "grad_norm": 0.6141477823257446, + "learning_rate": 9.00847134789411e-05, + "loss": 0.14525352478027342, + "mean_token_accuracy": 0.9490846493840217, + "num_tokens": 8515955.0, + "step": 2750 + }, + { + "entropy": 0.2143413795530796, + "epoch": 6.5423728813559325, + "grad_norm": 0.612184464931488, + "learning_rate": 8.502190224376661e-05, + "loss": 0.14751774787902833, + "mean_token_accuracy": 0.9478235357999801, + "num_tokens": 8672361.0, + "step": 2800 + }, + { + "entropy": 0.21027401253581046, + "epoch": 6.659263588544711, + "grad_norm": 0.644019365310669, + "learning_rate": 8.003690107962787e-05, + "loss": 0.1495517921447754, + "mean_token_accuracy": 0.9470956793427467, + "num_tokens": 8827819.0, + "step": 2850 + }, + { + "entropy": 0.2109992092102766, + "epoch": 6.776154295733489, + "grad_norm": 0.5456500053405762, + "learning_rate": 7.513799841659927e-05, + "loss": 0.14893126487731934, + "mean_token_accuracy": 0.9479587042331695, + "num_tokens": 8979535.0, + "step": 2900 + }, + { + "entropy": 0.20949276342988013, + "epoch": 6.893045002922268, + "grad_norm": 0.6803008913993835, + "learning_rate": 7.033333953104615e-05, + "loss": 0.1461741542816162, + "mean_token_accuracy": 0.9480843073129654, + "num_tokens": 9134134.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.4759114793949313, + "eval_loss": 0.9827883839607239, + "eval_mean_token_accuracy": 0.7945680889216337, + "eval_num_tokens": 9279501.0, + "eval_runtime": 36.4188, + "eval_samples_per_second": 33.801, + "eval_steps_per_second": 4.229, + "step": 2996 + }, + { + "entropy": 0.20382985060838, + "epoch": 7.009351256575102, + "grad_norm": 0.42079976201057434, + "learning_rate": 6.563091300268857e-05, + "loss": 0.13976585388183593, + "mean_token_accuracy": 0.9511888485458029, + "num_tokens": 9290416.0, + "step": 3000 + }, + { + "entropy": 0.14199006125330926, + "epoch": 7.1262419637638805, + "grad_norm": 0.6262232065200806, + "learning_rate": 6.103853743220019e-05, + "loss": 0.08045654296875, + "mean_token_accuracy": 0.9727814987301826, + "num_tokens": 9444705.0, + "step": 3050 + }, + { + "entropy": 0.13681559398770332, + "epoch": 7.243132670952659, + "grad_norm": 0.6460694074630737, + "learning_rate": 5.6563848441427006e-05, + "loss": 0.0795314359664917, + "mean_token_accuracy": 0.973130469918251, + "num_tokens": 9601476.0, + "step": 3100 + }, + { + "entropy": 0.13472390301525594, + "epoch": 7.360023378141438, + "grad_norm": 0.4053943157196045, + "learning_rate": 5.221428597784018e-05, + "loss": 0.07887161254882813, + "mean_token_accuracy": 0.9736681738495827, + "num_tokens": 9757913.0, + "step": 3150 + }, + { + "entropy": 0.14234945647418498, + "epoch": 7.4769140853302165, + "grad_norm": 0.46239086985588074, + "learning_rate": 4.7997081944331205e-05, + "loss": 0.08276874542236329, + "mean_token_accuracy": 0.9716356429457664, + "num_tokens": 9905790.0, + "step": 3200 + }, + { + "entropy": 0.13719632107764482, + "epoch": 7.593804792518995, + "grad_norm": 0.49838459491729736, + "learning_rate": 4.3919248174917856e-05, + "loss": 0.08003978729248047, + "mean_token_accuracy": 0.9731476292014122, + "num_tokens": 10064234.0, + "step": 3250 + }, + { + "entropy": 0.13751005962491036, + "epoch": 7.710695499707773, + "grad_norm": 0.4959999620914459, + "learning_rate": 3.9987564776352335e-05, + "loss": 0.07964549541473388, + "mean_token_accuracy": 0.9722868010401726, + "num_tokens": 10221089.0, + "step": 3300 + }, + { + "entropy": 0.13423183601349592, + "epoch": 7.827586206896552, + "grad_norm": 0.4889160692691803, + "learning_rate": 3.620856885501646e-05, + "loss": 0.07869738101959228, + "mean_token_accuracy": 0.973022837638855, + "num_tokens": 10379655.0, + "step": 3350 + }, + { + "entropy": 0.13587324120104313, + "epoch": 7.94447691408533, + "grad_norm": 0.4851946234703064, + "learning_rate": 3.258854364784726e-05, + "loss": 0.08004307746887207, + "mean_token_accuracy": 0.9723974686861038, + "num_tokens": 10531301.0, + "step": 3400 + }, + { + "epoch": 8.0, + "eval_entropy": 0.4391631047640528, + "eval_loss": 1.1321114301681519, + "eval_mean_token_accuracy": 0.7924294758152652, + "eval_num_tokens": 10605144.0, + "eval_runtime": 37.126, + "eval_samples_per_second": 33.157, + "eval_steps_per_second": 4.148, + "step": 3424 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.6381092874231296e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..aeac0e1485a5dfce2a7f1c470c2ee7b77e4dcdac --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-3852/trainer_state.json @@ -0,0 +1,903 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 9.0, + "eval_steps": 500, + "global_step": 3852, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + }, + { + "entropy": 0.38985791001188097, + "epoch": 5.023378141437756, + "grad_norm": 0.7454882860183716, + "learning_rate": 0.00015375688837676246, + "loss": 0.32916854858398437, + "mean_token_accuracy": 0.8886270373310875, + "num_tokens": 6662015.0, + "step": 2150 + }, + { + "entropy": 0.30170531339943407, + "epoch": 5.140268848626534, + "grad_norm": 0.5329811573028564, + "learning_rate": 0.0001484396230918344, + "loss": 0.23112209320068358, + "mean_token_accuracy": 0.9198204749822616, + "num_tokens": 6817989.0, + "step": 2200 + }, + { + "entropy": 0.3075008810311556, + "epoch": 5.257159555815313, + "grad_norm": 0.6896093487739563, + "learning_rate": 0.00014309472490462715, + "loss": 0.2416652488708496, + "mean_token_accuracy": 0.9158366405963898, + "num_tokens": 6977021.0, + "step": 2250 + }, + { + "entropy": 0.31153449676930905, + "epoch": 5.374050263004091, + "grad_norm": 0.6930833458900452, + "learning_rate": 0.00013773108063651161, + "loss": 0.24753889083862304, + "mean_token_accuracy": 0.9136614826321602, + "num_tokens": 7133360.0, + "step": 2300 + }, + { + "entropy": 0.3154346436262131, + "epoch": 5.490940970192869, + "grad_norm": 0.6476582884788513, + "learning_rate": 0.0001323576082774734, + "loss": 0.24872392654418946, + "mean_token_accuracy": 0.9135914054512978, + "num_tokens": 7282372.0, + "step": 2350 + }, + { + "entropy": 0.313641587048769, + "epoch": 5.607831677381649, + "grad_norm": 0.6953482627868652, + "learning_rate": 0.0001269832421584059, + "loss": 0.24919464111328124, + "mean_token_accuracy": 0.9131662029027939, + "num_tokens": 7435766.0, + "step": 2400 + }, + { + "entropy": 0.3163894522190094, + "epoch": 5.724722384570427, + "grad_norm": 0.6673940420150757, + "learning_rate": 0.00012161691809623384, + "loss": 0.25111726760864256, + "mean_token_accuracy": 0.913602537214756, + "num_tokens": 7578408.0, + "step": 2450 + }, + { + "entropy": 0.3183368914574385, + "epoch": 5.841613091759205, + "grad_norm": 0.7174487709999084, + "learning_rate": 0.00011626755853656576, + "loss": 0.2518378448486328, + "mean_token_accuracy": 0.912000589966774, + "num_tokens": 7734599.0, + "step": 2500 + }, + { + "entropy": 0.3077564483135939, + "epoch": 5.958503798947984, + "grad_norm": 0.6662200689315796, + "learning_rate": 0.00011094405771857891, + "loss": 0.24354087829589843, + "mean_token_accuracy": 0.9140791621804237, + "num_tokens": 7898028.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5403795917699863, + "eval_loss": 0.8572548627853394, + "eval_mean_token_accuracy": 0.796143418395674, + "eval_num_tokens": 7953858.0, + "eval_runtime": 36.7176, + "eval_samples_per_second": 33.526, + "eval_steps_per_second": 4.194, + "step": 2568 + }, + { + "entropy": 0.25110363144071857, + "epoch": 6.074810052600818, + "grad_norm": 0.6466763019561768, + "learning_rate": 0.0001056552668868023, + "loss": 0.1814822769165039, + "mean_token_accuracy": 0.9370300626035911, + "num_tokens": 8052646.0, + "step": 2600 + }, + { + "entropy": 0.2063255251944065, + "epoch": 6.1917007597895966, + "grad_norm": 0.6950973272323608, + "learning_rate": 0.0001004099795743853, + "loss": 0.13960565567016603, + "mean_token_accuracy": 0.9511301657557487, + "num_tokens": 8212323.0, + "step": 2650 + }, + { + "entropy": 0.20775293290615082, + "epoch": 6.308591466978375, + "grad_norm": 0.660478413105011, + "learning_rate": 9.52169169823223e-05, + "loss": 0.14319146156311036, + "mean_token_accuracy": 0.9500676044821739, + "num_tokens": 8366715.0, + "step": 2700 + }, + { + "entropy": 0.20977566957473756, + "epoch": 6.425482174167154, + "grad_norm": 0.6141477823257446, + "learning_rate": 9.00847134789411e-05, + "loss": 0.14525352478027342, + "mean_token_accuracy": 0.9490846493840217, + "num_tokens": 8515955.0, + "step": 2750 + }, + { + "entropy": 0.2143413795530796, + "epoch": 6.5423728813559325, + "grad_norm": 0.612184464931488, + "learning_rate": 8.502190224376661e-05, + "loss": 0.14751774787902833, + "mean_token_accuracy": 0.9478235357999801, + "num_tokens": 8672361.0, + "step": 2800 + }, + { + "entropy": 0.21027401253581046, + "epoch": 6.659263588544711, + "grad_norm": 0.644019365310669, + "learning_rate": 8.003690107962787e-05, + "loss": 0.1495517921447754, + "mean_token_accuracy": 0.9470956793427467, + "num_tokens": 8827819.0, + "step": 2850 + }, + { + "entropy": 0.2109992092102766, + "epoch": 6.776154295733489, + "grad_norm": 0.5456500053405762, + "learning_rate": 7.513799841659927e-05, + "loss": 0.14893126487731934, + "mean_token_accuracy": 0.9479587042331695, + "num_tokens": 8979535.0, + "step": 2900 + }, + { + "entropy": 0.20949276342988013, + "epoch": 6.893045002922268, + "grad_norm": 0.6803008913993835, + "learning_rate": 7.033333953104615e-05, + "loss": 0.1461741542816162, + "mean_token_accuracy": 0.9480843073129654, + "num_tokens": 9134134.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.4759114793949313, + "eval_loss": 0.9827883839607239, + "eval_mean_token_accuracy": 0.7945680889216337, + "eval_num_tokens": 9279501.0, + "eval_runtime": 36.4188, + "eval_samples_per_second": 33.801, + "eval_steps_per_second": 4.229, + "step": 2996 + }, + { + "entropy": 0.20382985060838, + "epoch": 7.009351256575102, + "grad_norm": 0.42079976201057434, + "learning_rate": 6.563091300268857e-05, + "loss": 0.13976585388183593, + "mean_token_accuracy": 0.9511888485458029, + "num_tokens": 9290416.0, + "step": 3000 + }, + { + "entropy": 0.14199006125330926, + "epoch": 7.1262419637638805, + "grad_norm": 0.6262232065200806, + "learning_rate": 6.103853743220019e-05, + "loss": 0.08045654296875, + "mean_token_accuracy": 0.9727814987301826, + "num_tokens": 9444705.0, + "step": 3050 + }, + { + "entropy": 0.13681559398770332, + "epoch": 7.243132670952659, + "grad_norm": 0.6460694074630737, + "learning_rate": 5.6563848441427006e-05, + "loss": 0.0795314359664917, + "mean_token_accuracy": 0.973130469918251, + "num_tokens": 9601476.0, + "step": 3100 + }, + { + "entropy": 0.13472390301525594, + "epoch": 7.360023378141438, + "grad_norm": 0.4053943157196045, + "learning_rate": 5.221428597784018e-05, + "loss": 0.07887161254882813, + "mean_token_accuracy": 0.9736681738495827, + "num_tokens": 9757913.0, + "step": 3150 + }, + { + "entropy": 0.14234945647418498, + "epoch": 7.4769140853302165, + "grad_norm": 0.46239086985588074, + "learning_rate": 4.7997081944331205e-05, + "loss": 0.08276874542236329, + "mean_token_accuracy": 0.9716356429457664, + "num_tokens": 9905790.0, + "step": 3200 + }, + { + "entropy": 0.13719632107764482, + "epoch": 7.593804792518995, + "grad_norm": 0.49838459491729736, + "learning_rate": 4.3919248174917856e-05, + "loss": 0.08003978729248047, + "mean_token_accuracy": 0.9731476292014122, + "num_tokens": 10064234.0, + "step": 3250 + }, + { + "entropy": 0.13751005962491036, + "epoch": 7.710695499707773, + "grad_norm": 0.4959999620914459, + "learning_rate": 3.9987564776352335e-05, + "loss": 0.07964549541473388, + "mean_token_accuracy": 0.9722868010401726, + "num_tokens": 10221089.0, + "step": 3300 + }, + { + "entropy": 0.13423183601349592, + "epoch": 7.827586206896552, + "grad_norm": 0.4889160692691803, + "learning_rate": 3.620856885501646e-05, + "loss": 0.07869738101959228, + "mean_token_accuracy": 0.973022837638855, + "num_tokens": 10379655.0, + "step": 3350 + }, + { + "entropy": 0.13587324120104313, + "epoch": 7.94447691408533, + "grad_norm": 0.4851946234703064, + "learning_rate": 3.258854364784726e-05, + "loss": 0.08004307746887207, + "mean_token_accuracy": 0.9723974686861038, + "num_tokens": 10531301.0, + "step": 3400 + }, + { + "epoch": 8.0, + "eval_entropy": 0.4391631047640528, + "eval_loss": 1.1321114301681519, + "eval_mean_token_accuracy": 0.7924294758152652, + "eval_num_tokens": 10605144.0, + "eval_runtime": 37.126, + "eval_samples_per_second": 33.157, + "eval_steps_per_second": 4.148, + "step": 3424 + }, + { + "entropy": 0.12316639324528487, + "epoch": 8.060783167738165, + "grad_norm": 0.3510015606880188, + "learning_rate": 2.9133508075364238e-05, + "loss": 0.06677456378936768, + "mean_token_accuracy": 0.9770565955483135, + "num_tokens": 10686079.0, + "step": 3450 + }, + { + "entropy": 0.11033655313774943, + "epoch": 8.177673874926944, + "grad_norm": 0.27871957421302795, + "learning_rate": 2.5849206734169147e-05, + "loss": 0.05629729747772217, + "mean_token_accuracy": 0.9804861381649971, + "num_tokens": 10838236.0, + "step": 3500 + }, + { + "entropy": 0.10826377695426345, + "epoch": 8.294564582115722, + "grad_norm": 0.44871190190315247, + "learning_rate": 2.2741100345556444e-05, + "loss": 0.05603146076202393, + "mean_token_accuracy": 0.9803140532970428, + "num_tokens": 10989653.0, + "step": 3550 + }, + { + "entropy": 0.10614268690347671, + "epoch": 8.4114552893045, + "grad_norm": 0.25647690892219543, + "learning_rate": 1.9814356676115743e-05, + "loss": 0.0554099178314209, + "mean_token_accuracy": 0.9802037307620048, + "num_tokens": 11144096.0, + "step": 3600 + }, + { + "entropy": 0.10369085857644678, + "epoch": 8.528345996493279, + "grad_norm": 0.32260316610336304, + "learning_rate": 1.7073841945422747e-05, + "loss": 0.05409110546112061, + "mean_token_accuracy": 0.980969468653202, + "num_tokens": 11303465.0, + "step": 3650 + }, + { + "entropy": 0.10663174549117685, + "epoch": 8.645236703682057, + "grad_norm": 0.33514541387557983, + "learning_rate": 1.4524112735104158e-05, + "loss": 0.056715693473815915, + "mean_token_accuracy": 0.9798875021934509, + "num_tokens": 11455533.0, + "step": 3700 + }, + { + "entropy": 0.1048200499266386, + "epoch": 8.762127410870836, + "grad_norm": 0.3340476453304291, + "learning_rate": 1.2169408412729229e-05, + "loss": 0.05345354557037354, + "mean_token_accuracy": 0.9803790295124054, + "num_tokens": 11618619.0, + "step": 3750 + }, + { + "entropy": 0.10449992345646024, + "epoch": 8.879018118059614, + "grad_norm": 0.30406567454338074, + "learning_rate": 1.0013644083125266e-05, + "loss": 0.054877762794494626, + "mean_token_accuracy": 0.9807043489813805, + "num_tokens": 11774627.0, + "step": 3800 + }, + { + "entropy": 0.10615571580827236, + "epoch": 8.995908825248392, + "grad_norm": 0.36648011207580566, + "learning_rate": 8.060404078835946e-06, + "loss": 0.056678996086120606, + "mean_token_accuracy": 0.9797374567389489, + "num_tokens": 11925408.0, + "step": 3850 + }, + { + "epoch": 9.0, + "eval_entropy": 0.4037178186820699, + "eval_loss": 1.254728078842163, + "eval_mean_token_accuracy": 0.7945115701718763, + "eval_num_tokens": 11930787.0, + "eval_runtime": 36.9056, + "eval_samples_per_second": 33.355, + "eval_steps_per_second": 4.173, + "step": 3852 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.84246098288768e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c6fa236661f43829aae12f37410d042a38dd8dcd --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-428/trainer_state.json @@ -0,0 +1,125 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0, + "eval_steps": 500, + "global_step": 428, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.053230260741376e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..307d039bfa0c3dffaa766861956197041f3bc819 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-4280/trainer_state.json @@ -0,0 +1,994 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 10.0, + "eval_steps": 500, + "global_step": 4280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + }, + { + "entropy": 0.5709320336730037, + "epoch": 2.102863822326125, + "grad_norm": 0.4339260458946228, + "learning_rate": 0.0002540334434848298, + "loss": 0.5152394485473633, + "mean_token_accuracy": 0.8391561669919958, + "num_tokens": 2795200.0, + "step": 900 + }, + { + "entropy": 0.5768725140392781, + "epoch": 2.2197545295149035, + "grad_norm": 0.6345565319061279, + "learning_rate": 0.00025191790796680617, + "loss": 0.5241253662109375, + "mean_token_accuracy": 0.8377170553803444, + "num_tokens": 2950534.0, + "step": 950 + }, + { + "entropy": 0.570428695231676, + "epoch": 2.3366452367036823, + "grad_norm": 0.5838360786437988, + "learning_rate": 0.00024960268892902876, + "loss": 0.5187693405151367, + "mean_token_accuracy": 0.8382867723703384, + "num_tokens": 3101471.0, + "step": 1000 + }, + { + "entropy": 0.5713468404114246, + "epoch": 2.4535359438924607, + "grad_norm": 0.5641153454780579, + "learning_rate": 0.0002470916358251826, + "loss": 0.5234761810302735, + "mean_token_accuracy": 0.8382993021607399, + "num_tokens": 3258554.0, + "step": 1050 + }, + { + "entropy": 0.5645828676223755, + "epoch": 2.570426651081239, + "grad_norm": 0.504467785358429, + "learning_rate": 0.0002443889237170939, + "loss": 0.5156657409667968, + "mean_token_accuracy": 0.8391393664479255, + "num_tokens": 3416088.0, + "step": 1100 + }, + { + "entropy": 0.5659542319178581, + "epoch": 2.6873173582700174, + "grad_norm": 0.5225860476493835, + "learning_rate": 0.00024149904633296492, + "loss": 0.5180046844482422, + "mean_token_accuracy": 0.8389606699347496, + "num_tokens": 3567251.0, + "step": 1150 + }, + { + "entropy": 0.5713718670606613, + "epoch": 2.8042080654587958, + "grad_norm": 0.4986768066883087, + "learning_rate": 0.0002384268085957702, + "loss": 0.524637565612793, + "mean_token_accuracy": 0.8381215763092041, + "num_tokens": 3719464.0, + "step": 1200 + }, + { + "entropy": 0.5710382495820522, + "epoch": 2.9210987726475746, + "grad_norm": 0.5548401474952698, + "learning_rate": 0.00023517731863423788, + "loss": 0.5251939773559571, + "mean_token_accuracy": 0.8378310143947602, + "num_tokens": 3874423.0, + "step": 1250 + }, + { + "epoch": 3.0, + "eval_entropy": 0.6970683218209774, + "eval_loss": 0.7369498014450073, + "eval_mean_token_accuracy": 0.7919643076209278, + "eval_num_tokens": 3976929.0, + "eval_runtime": 36.9722, + "eval_samples_per_second": 33.295, + "eval_steps_per_second": 4.165, + "step": 1284 + }, + { + "entropy": 0.5407733828877684, + "epoch": 3.037405026300409, + "grad_norm": 0.6430732607841492, + "learning_rate": 0.00023175597928969878, + "loss": 0.49219757080078125, + "mean_token_accuracy": 0.8467060854686564, + "num_tokens": 4027609.0, + "step": 1300 + }, + { + "entropy": 0.4786584210395813, + "epoch": 3.1542957334891875, + "grad_norm": 0.6474543809890747, + "learning_rate": 0.00022816847913292434, + "loss": 0.4224985122680664, + "mean_token_accuracy": 0.8628773537278175, + "num_tokens": 4180560.0, + "step": 1350 + }, + { + "entropy": 0.49172899857163427, + "epoch": 3.2711864406779663, + "grad_norm": 0.549326479434967, + "learning_rate": 0.00022442078300588989, + "loss": 0.43718509674072265, + "mean_token_accuracy": 0.8583108323812485, + "num_tokens": 4329954.0, + "step": 1400 + }, + { + "entropy": 0.4829080794751644, + "epoch": 3.3880771478667446, + "grad_norm": 0.5274888873100281, + "learning_rate": 0.00022051912210418922, + "loss": 0.4315507125854492, + "mean_token_accuracy": 0.8612637943029404, + "num_tokens": 4484354.0, + "step": 1450 + }, + { + "entropy": 0.4855460473895073, + "epoch": 3.504967855055523, + "grad_norm": 0.5846306681632996, + "learning_rate": 0.00021646998361658985, + "loss": 0.43724521636962893, + "mean_token_accuracy": 0.8589270997047425, + "num_tokens": 4638919.0, + "step": 1500 + }, + { + "entropy": 0.4905838692188263, + "epoch": 3.6218585622443014, + "grad_norm": 0.5149123072624207, + "learning_rate": 0.00021228009993895495, + "loss": 0.44197372436523436, + "mean_token_accuracy": 0.8584243017435074, + "num_tokens": 4795540.0, + "step": 1550 + }, + { + "entropy": 0.4994237454235554, + "epoch": 3.73874926943308, + "grad_norm": 0.5417225956916809, + "learning_rate": 0.0002079564374804665, + "loss": 0.4491108322143555, + "mean_token_accuracy": 0.8556393891572952, + "num_tokens": 4951177.0, + "step": 1600 + }, + { + "entropy": 0.49903531357645986, + "epoch": 3.8556399766218585, + "grad_norm": 0.5086734890937805, + "learning_rate": 0.00020350618508076015, + "loss": 0.4518918228149414, + "mean_token_accuracy": 0.8549707370996475, + "num_tokens": 5104814.0, + "step": 1650 + }, + { + "entropy": 0.4859523306787014, + "epoch": 3.972530683810637, + "grad_norm": 0.5446717739105225, + "learning_rate": 0.00019893674205723094, + "loss": 0.4410313415527344, + "mean_token_accuracy": 0.8587654489278793, + "num_tokens": 5265185.0, + "step": 1700 + }, + { + "epoch": 4.0, + "eval_entropy": 0.6577693150415049, + "eval_loss": 0.7380754947662354, + "eval_mean_token_accuracy": 0.8039549565934515, + "eval_num_tokens": 5302572.0, + "eval_runtime": 36.5757, + "eval_samples_per_second": 33.656, + "eval_steps_per_second": 4.21, + "step": 1712 + }, + { + "entropy": 0.41945636961328325, + "epoch": 4.088836937463472, + "grad_norm": 0.5925881266593933, + "learning_rate": 0.00019425570590238332, + "loss": 0.3575031280517578, + "mean_token_accuracy": 0.8812823918596584, + "num_tokens": 5419002.0, + "step": 1750 + }, + { + "entropy": 0.40106018364429474, + "epoch": 4.20572764465225, + "grad_norm": 0.7094829082489014, + "learning_rate": 0.0001894708596516806, + "loss": 0.3372145080566406, + "mean_token_accuracy": 0.8856011089682579, + "num_tokens": 5572969.0, + "step": 1800 + }, + { + "entropy": 0.4121378688514233, + "epoch": 4.322618351841029, + "grad_norm": 0.6077721118927002, + "learning_rate": 0.00018459015894289695, + "loss": 0.34956165313720705, + "mean_token_accuracy": 0.8818695649504662, + "num_tokens": 5733805.0, + "step": 1850 + }, + { + "entropy": 0.40683484479784965, + "epoch": 4.439509059029807, + "grad_norm": 0.5614883303642273, + "learning_rate": 0.00017962171878848807, + "loss": 0.34749691009521483, + "mean_token_accuracy": 0.882930768430233, + "num_tokens": 5888694.0, + "step": 1900 + }, + { + "entropy": 0.4100585526227951, + "epoch": 4.556399766218585, + "grad_norm": 0.5506443977355957, + "learning_rate": 0.00017457380008297383, + "loss": 0.3501173782348633, + "mean_token_accuracy": 0.8827524068951607, + "num_tokens": 6042655.0, + "step": 1950 + }, + { + "entropy": 0.41173958137631417, + "epoch": 4.673290473407365, + "grad_norm": 0.5409566164016724, + "learning_rate": 0.00016945479586776672, + "loss": 0.3541391754150391, + "mean_token_accuracy": 0.8808356913924217, + "num_tokens": 6203980.0, + "step": 2000 + }, + { + "entropy": 0.4158783262968063, + "epoch": 4.790181180596143, + "grad_norm": 0.572618842124939, + "learning_rate": 0.00016427321737628295, + "loss": 0.3536404800415039, + "mean_token_accuracy": 0.880456589460373, + "num_tokens": 6354604.0, + "step": 2050 + }, + { + "entropy": 0.408166281580925, + "epoch": 4.907071887784921, + "grad_norm": 0.5987762808799744, + "learning_rate": 0.00015903767988253897, + "loss": 0.3518599319458008, + "mean_token_accuracy": 0.8815453574061394, + "num_tokens": 6507484.0, + "step": 2100 + }, + { + "epoch": 5.0, + "eval_entropy": 0.6047254425364655, + "eval_loss": 0.7816920876502991, + "eval_mean_token_accuracy": 0.7987685733801358, + "eval_num_tokens": 6628215.0, + "eval_runtime": 36.9374, + "eval_samples_per_second": 33.327, + "eval_steps_per_second": 4.169, + "step": 2140 + }, + { + "entropy": 0.38985791001188097, + "epoch": 5.023378141437756, + "grad_norm": 0.7454882860183716, + "learning_rate": 0.00015375688837676246, + "loss": 0.32916854858398437, + "mean_token_accuracy": 0.8886270373310875, + "num_tokens": 6662015.0, + "step": 2150 + }, + { + "entropy": 0.30170531339943407, + "epoch": 5.140268848626534, + "grad_norm": 0.5329811573028564, + "learning_rate": 0.0001484396230918344, + "loss": 0.23112209320068358, + "mean_token_accuracy": 0.9198204749822616, + "num_tokens": 6817989.0, + "step": 2200 + }, + { + "entropy": 0.3075008810311556, + "epoch": 5.257159555815313, + "grad_norm": 0.6896093487739563, + "learning_rate": 0.00014309472490462715, + "loss": 0.2416652488708496, + "mean_token_accuracy": 0.9158366405963898, + "num_tokens": 6977021.0, + "step": 2250 + }, + { + "entropy": 0.31153449676930905, + "epoch": 5.374050263004091, + "grad_norm": 0.6930833458900452, + "learning_rate": 0.00013773108063651161, + "loss": 0.24753889083862304, + "mean_token_accuracy": 0.9136614826321602, + "num_tokens": 7133360.0, + "step": 2300 + }, + { + "entropy": 0.3154346436262131, + "epoch": 5.490940970192869, + "grad_norm": 0.6476582884788513, + "learning_rate": 0.0001323576082774734, + "loss": 0.24872392654418946, + "mean_token_accuracy": 0.9135914054512978, + "num_tokens": 7282372.0, + "step": 2350 + }, + { + "entropy": 0.313641587048769, + "epoch": 5.607831677381649, + "grad_norm": 0.6953482627868652, + "learning_rate": 0.0001269832421584059, + "loss": 0.24919464111328124, + "mean_token_accuracy": 0.9131662029027939, + "num_tokens": 7435766.0, + "step": 2400 + }, + { + "entropy": 0.3163894522190094, + "epoch": 5.724722384570427, + "grad_norm": 0.6673940420150757, + "learning_rate": 0.00012161691809623384, + "loss": 0.25111726760864256, + "mean_token_accuracy": 0.913602537214756, + "num_tokens": 7578408.0, + "step": 2450 + }, + { + "entropy": 0.3183368914574385, + "epoch": 5.841613091759205, + "grad_norm": 0.7174487709999084, + "learning_rate": 0.00011626755853656576, + "loss": 0.2518378448486328, + "mean_token_accuracy": 0.912000589966774, + "num_tokens": 7734599.0, + "step": 2500 + }, + { + "entropy": 0.3077564483135939, + "epoch": 5.958503798947984, + "grad_norm": 0.6662200689315796, + "learning_rate": 0.00011094405771857891, + "loss": 0.24354087829589843, + "mean_token_accuracy": 0.9140791621804237, + "num_tokens": 7898028.0, + "step": 2550 + }, + { + "epoch": 6.0, + "eval_entropy": 0.5403795917699863, + "eval_loss": 0.8572548627853394, + "eval_mean_token_accuracy": 0.796143418395674, + "eval_num_tokens": 7953858.0, + "eval_runtime": 36.7176, + "eval_samples_per_second": 33.526, + "eval_steps_per_second": 4.194, + "step": 2568 + }, + { + "entropy": 0.25110363144071857, + "epoch": 6.074810052600818, + "grad_norm": 0.6466763019561768, + "learning_rate": 0.0001056552668868023, + "loss": 0.1814822769165039, + "mean_token_accuracy": 0.9370300626035911, + "num_tokens": 8052646.0, + "step": 2600 + }, + { + "entropy": 0.2063255251944065, + "epoch": 6.1917007597895966, + "grad_norm": 0.6950973272323608, + "learning_rate": 0.0001004099795743853, + "loss": 0.13960565567016603, + "mean_token_accuracy": 0.9511301657557487, + "num_tokens": 8212323.0, + "step": 2650 + }, + { + "entropy": 0.20775293290615082, + "epoch": 6.308591466978375, + "grad_norm": 0.660478413105011, + "learning_rate": 9.52169169823223e-05, + "loss": 0.14319146156311036, + "mean_token_accuracy": 0.9500676044821739, + "num_tokens": 8366715.0, + "step": 2700 + }, + { + "entropy": 0.20977566957473756, + "epoch": 6.425482174167154, + "grad_norm": 0.6141477823257446, + "learning_rate": 9.00847134789411e-05, + "loss": 0.14525352478027342, + "mean_token_accuracy": 0.9490846493840217, + "num_tokens": 8515955.0, + "step": 2750 + }, + { + "entropy": 0.2143413795530796, + "epoch": 6.5423728813559325, + "grad_norm": 0.612184464931488, + "learning_rate": 8.502190224376661e-05, + "loss": 0.14751774787902833, + "mean_token_accuracy": 0.9478235357999801, + "num_tokens": 8672361.0, + "step": 2800 + }, + { + "entropy": 0.21027401253581046, + "epoch": 6.659263588544711, + "grad_norm": 0.644019365310669, + "learning_rate": 8.003690107962787e-05, + "loss": 0.1495517921447754, + "mean_token_accuracy": 0.9470956793427467, + "num_tokens": 8827819.0, + "step": 2850 + }, + { + "entropy": 0.2109992092102766, + "epoch": 6.776154295733489, + "grad_norm": 0.5456500053405762, + "learning_rate": 7.513799841659927e-05, + "loss": 0.14893126487731934, + "mean_token_accuracy": 0.9479587042331695, + "num_tokens": 8979535.0, + "step": 2900 + }, + { + "entropy": 0.20949276342988013, + "epoch": 6.893045002922268, + "grad_norm": 0.6803008913993835, + "learning_rate": 7.033333953104615e-05, + "loss": 0.1461741542816162, + "mean_token_accuracy": 0.9480843073129654, + "num_tokens": 9134134.0, + "step": 2950 + }, + { + "epoch": 7.0, + "eval_entropy": 0.4759114793949313, + "eval_loss": 0.9827883839607239, + "eval_mean_token_accuracy": 0.7945680889216337, + "eval_num_tokens": 9279501.0, + "eval_runtime": 36.4188, + "eval_samples_per_second": 33.801, + "eval_steps_per_second": 4.229, + "step": 2996 + }, + { + "entropy": 0.20382985060838, + "epoch": 7.009351256575102, + "grad_norm": 0.42079976201057434, + "learning_rate": 6.563091300268857e-05, + "loss": 0.13976585388183593, + "mean_token_accuracy": 0.9511888485458029, + "num_tokens": 9290416.0, + "step": 3000 + }, + { + "entropy": 0.14199006125330926, + "epoch": 7.1262419637638805, + "grad_norm": 0.6262232065200806, + "learning_rate": 6.103853743220019e-05, + "loss": 0.08045654296875, + "mean_token_accuracy": 0.9727814987301826, + "num_tokens": 9444705.0, + "step": 3050 + }, + { + "entropy": 0.13681559398770332, + "epoch": 7.243132670952659, + "grad_norm": 0.6460694074630737, + "learning_rate": 5.6563848441427006e-05, + "loss": 0.0795314359664917, + "mean_token_accuracy": 0.973130469918251, + "num_tokens": 9601476.0, + "step": 3100 + }, + { + "entropy": 0.13472390301525594, + "epoch": 7.360023378141438, + "grad_norm": 0.4053943157196045, + "learning_rate": 5.221428597784018e-05, + "loss": 0.07887161254882813, + "mean_token_accuracy": 0.9736681738495827, + "num_tokens": 9757913.0, + "step": 3150 + }, + { + "entropy": 0.14234945647418498, + "epoch": 7.4769140853302165, + "grad_norm": 0.46239086985588074, + "learning_rate": 4.7997081944331205e-05, + "loss": 0.08276874542236329, + "mean_token_accuracy": 0.9716356429457664, + "num_tokens": 9905790.0, + "step": 3200 + }, + { + "entropy": 0.13719632107764482, + "epoch": 7.593804792518995, + "grad_norm": 0.49838459491729736, + "learning_rate": 4.3919248174917856e-05, + "loss": 0.08003978729248047, + "mean_token_accuracy": 0.9731476292014122, + "num_tokens": 10064234.0, + "step": 3250 + }, + { + "entropy": 0.13751005962491036, + "epoch": 7.710695499707773, + "grad_norm": 0.4959999620914459, + "learning_rate": 3.9987564776352335e-05, + "loss": 0.07964549541473388, + "mean_token_accuracy": 0.9722868010401726, + "num_tokens": 10221089.0, + "step": 3300 + }, + { + "entropy": 0.13423183601349592, + "epoch": 7.827586206896552, + "grad_norm": 0.4889160692691803, + "learning_rate": 3.620856885501646e-05, + "loss": 0.07869738101959228, + "mean_token_accuracy": 0.973022837638855, + "num_tokens": 10379655.0, + "step": 3350 + }, + { + "entropy": 0.13587324120104313, + "epoch": 7.94447691408533, + "grad_norm": 0.4851946234703064, + "learning_rate": 3.258854364784726e-05, + "loss": 0.08004307746887207, + "mean_token_accuracy": 0.9723974686861038, + "num_tokens": 10531301.0, + "step": 3400 + }, + { + "epoch": 8.0, + "eval_entropy": 0.4391631047640528, + "eval_loss": 1.1321114301681519, + "eval_mean_token_accuracy": 0.7924294758152652, + "eval_num_tokens": 10605144.0, + "eval_runtime": 37.126, + "eval_samples_per_second": 33.157, + "eval_steps_per_second": 4.148, + "step": 3424 + }, + { + "entropy": 0.12316639324528487, + "epoch": 8.060783167738165, + "grad_norm": 0.3510015606880188, + "learning_rate": 2.9133508075364238e-05, + "loss": 0.06677456378936768, + "mean_token_accuracy": 0.9770565955483135, + "num_tokens": 10686079.0, + "step": 3450 + }, + { + "entropy": 0.11033655313774943, + "epoch": 8.177673874926944, + "grad_norm": 0.27871957421302795, + "learning_rate": 2.5849206734169147e-05, + "loss": 0.05629729747772217, + "mean_token_accuracy": 0.9804861381649971, + "num_tokens": 10838236.0, + "step": 3500 + }, + { + "entropy": 0.10826377695426345, + "epoch": 8.294564582115722, + "grad_norm": 0.44871190190315247, + "learning_rate": 2.2741100345556444e-05, + "loss": 0.05603146076202393, + "mean_token_accuracy": 0.9803140532970428, + "num_tokens": 10989653.0, + "step": 3550 + }, + { + "entropy": 0.10614268690347671, + "epoch": 8.4114552893045, + "grad_norm": 0.25647690892219543, + "learning_rate": 1.9814356676115743e-05, + "loss": 0.0554099178314209, + "mean_token_accuracy": 0.9802037307620048, + "num_tokens": 11144096.0, + "step": 3600 + }, + { + "entropy": 0.10369085857644678, + "epoch": 8.528345996493279, + "grad_norm": 0.32260316610336304, + "learning_rate": 1.7073841945422747e-05, + "loss": 0.05409110546112061, + "mean_token_accuracy": 0.980969468653202, + "num_tokens": 11303465.0, + "step": 3650 + }, + { + "entropy": 0.10663174549117685, + "epoch": 8.645236703682057, + "grad_norm": 0.33514541387557983, + "learning_rate": 1.4524112735104158e-05, + "loss": 0.056715693473815915, + "mean_token_accuracy": 0.9798875021934509, + "num_tokens": 11455533.0, + "step": 3700 + }, + { + "entropy": 0.1048200499266386, + "epoch": 8.762127410870836, + "grad_norm": 0.3340476453304291, + "learning_rate": 1.2169408412729229e-05, + "loss": 0.05345354557037354, + "mean_token_accuracy": 0.9803790295124054, + "num_tokens": 11618619.0, + "step": 3750 + }, + { + "entropy": 0.10449992345646024, + "epoch": 8.879018118059614, + "grad_norm": 0.30406567454338074, + "learning_rate": 1.0013644083125266e-05, + "loss": 0.054877762794494626, + "mean_token_accuracy": 0.9807043489813805, + "num_tokens": 11774627.0, + "step": 3800 + }, + { + "entropy": 0.10615571580827236, + "epoch": 8.995908825248392, + "grad_norm": 0.36648011207580566, + "learning_rate": 8.060404078835946e-06, + "loss": 0.056678996086120606, + "mean_token_accuracy": 0.9797374567389489, + "num_tokens": 11925408.0, + "step": 3850 + }, + { + "epoch": 9.0, + "eval_entropy": 0.4037178186820699, + "eval_loss": 1.254728078842163, + "eval_mean_token_accuracy": 0.7945115701718763, + "eval_num_tokens": 11930787.0, + "eval_runtime": 36.9056, + "eval_samples_per_second": 33.355, + "eval_steps_per_second": 4.173, + "step": 3852 + }, + { + "entropy": 0.09968331250263818, + "epoch": 9.112215078901228, + "grad_norm": 0.21597710251808167, + "learning_rate": 6.312936000545962e-06, + "loss": 0.04916244506835937, + "mean_token_accuracy": 0.9826284425342502, + "num_tokens": 12078366.0, + "step": 3900 + }, + { + "entropy": 0.1036025501601398, + "epoch": 9.229105786090006, + "grad_norm": 0.1738789677619934, + "learning_rate": 4.774145317381034e-06, + "loss": 0.05066513061523437, + "mean_token_accuracy": 0.9816445669531823, + "num_tokens": 12229118.0, + "step": 3950 + }, + { + "entropy": 0.09634713580831886, + "epoch": 9.345996493278784, + "grad_norm": 0.15528039634227753, + "learning_rate": 3.4465905360609734e-06, + "loss": 0.048097033500671384, + "mean_token_accuracy": 0.9828672608733178, + "num_tokens": 12385835.0, + "step": 4000 + }, + { + "entropy": 0.09529286917299032, + "epoch": 9.462887200467563, + "grad_norm": 0.3298656940460205, + "learning_rate": 2.3324789469378376e-06, + "loss": 0.04854735374450683, + "mean_token_accuracy": 0.9828567364811898, + "num_tokens": 12543991.0, + "step": 4050 + }, + { + "entropy": 0.09487062746658921, + "epoch": 9.579777907656341, + "grad_norm": 0.1514902114868164, + "learning_rate": 1.4336629539926327e-06, + "loss": 0.04930572509765625, + "mean_token_accuracy": 0.982597168982029, + "num_tokens": 12701866.0, + "step": 4100 + }, + { + "entropy": 0.09462659215554595, + "epoch": 9.69666861484512, + "grad_norm": 0.20895801484584808, + "learning_rate": 7.516369948918152e-07, + "loss": 0.048983674049377444, + "mean_token_accuracy": 0.9824120745062828, + "num_tokens": 12858675.0, + "step": 4150 + }, + { + "entropy": 0.09866151874884962, + "epoch": 9.813559322033898, + "grad_norm": 0.19578123092651367, + "learning_rate": 2.8753505622533914e-07, + "loss": 0.050036773681640626, + "mean_token_accuracy": 0.9818735978007317, + "num_tokens": 13010982.0, + "step": 4200 + }, + { + "entropy": 0.10088366318494081, + "epoch": 9.930450029222676, + "grad_norm": 0.26500314474105835, + "learning_rate": 4.2128788056935204e-08, + "loss": 0.0506698226928711, + "mean_token_accuracy": 0.9811591205000877, + "num_tokens": 13161041.0, + "step": 4250 + }, + { + "epoch": 10.0, + "eval_entropy": 0.39390789359420925, + "eval_loss": 1.3178749084472656, + "eval_mean_token_accuracy": 0.7940253568934156, + "eval_num_tokens": 13256430.0, + "eval_runtime": 36.3673, + "eval_samples_per_second": 33.849, + "eval_steps_per_second": 4.235, + "step": 4280 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 2.046123177338496e+17, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/README.md b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/adapter_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9e45544de727016be134f25e606b7b2b87c67549 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.050044417519510255, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "v_proj", + "o_proj", + "q_proj", + "gate_proj", + "k_proj", + "down_proj", + "up_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/chat_template.jinja b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/tokenizer_config.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/trainer_state.json b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..da46d88888cf4e6160b9e71d92c4f14b1cea021f --- /dev/null +++ b/DBCA_original_Estonian/Qwen3.5-2B-Base_original_features_structural_train_original_features_structural_test2/checkpoint-856/trainer_state.json @@ -0,0 +1,226 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 2.0, + "eval_steps": 500, + "global_step": 856, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 1.8448095142841339, + "epoch": 0.11689070718877849, + "grad_norm": 1.4738068580627441, + "learning_rate": 3.0183098859435753e-05, + "loss": 1.6948321533203126, + "mean_token_accuracy": 0.6315939916670322, + "num_tokens": 153689.0, + "step": 50 + }, + { + "entropy": 0.9560942512750625, + "epoch": 0.23378141437755698, + "grad_norm": 1.378090739250183, + "learning_rate": 6.098217932824775e-05, + "loss": 0.8888477325439453, + "mean_token_accuracy": 0.7586692428588867, + "num_tokens": 310548.0, + "step": 100 + }, + { + "entropy": 0.848925698697567, + "epoch": 0.3506721215663355, + "grad_norm": 0.959621250629425, + "learning_rate": 9.178125979705974e-05, + "loss": 0.7850731658935547, + "mean_token_accuracy": 0.7774798944592476, + "num_tokens": 468560.0, + "step": 150 + }, + { + "entropy": 0.811761694252491, + "epoch": 0.46756282875511396, + "grad_norm": 0.8867514133453369, + "learning_rate": 0.00012258034026587175, + "loss": 0.7446884155273438, + "mean_token_accuracy": 0.7892421445250511, + "num_tokens": 619552.0, + "step": 200 + }, + { + "entropy": 0.7709847274422645, + "epoch": 0.5844535359438925, + "grad_norm": 0.5822142362594604, + "learning_rate": 0.00015337942073468373, + "loss": 0.7130480194091797, + "mean_token_accuracy": 0.7964023387432099, + "num_tokens": 771972.0, + "step": 250 + }, + { + "entropy": 0.7401632118225098, + "epoch": 0.701344243132671, + "grad_norm": 0.7557511925697327, + "learning_rate": 0.00018417850120349574, + "loss": 0.6857429504394531, + "mean_token_accuracy": 0.8041065946221352, + "num_tokens": 928088.0, + "step": 300 + }, + { + "entropy": 0.7295274111628532, + "epoch": 0.8182349503214494, + "grad_norm": 0.6769705414772034, + "learning_rate": 0.00021497758167230772, + "loss": 0.6801216125488281, + "mean_token_accuracy": 0.8037989619374275, + "num_tokens": 1086836.0, + "step": 350 + }, + { + "entropy": 0.710282272696495, + "epoch": 0.9351256575102279, + "grad_norm": 0.832009494304657, + "learning_rate": 0.00024577666214111967, + "loss": 0.6587177276611328, + "mean_token_accuracy": 0.8084024578332901, + "num_tokens": 1241227.0, + "step": 400 + }, + { + "epoch": 1.0, + "eval_entropy": 0.848237401672772, + "eval_loss": 0.7832363247871399, + "eval_mean_token_accuracy": 0.7817642971292719, + "eval_num_tokens": 1325643.0, + "eval_runtime": 36.8595, + "eval_samples_per_second": 33.397, + "eval_steps_per_second": 4.178, + "step": 428 + }, + { + "entropy": 0.7039205249230466, + "epoch": 1.0514319111630626, + "grad_norm": 0.6213153600692749, + "learning_rate": 0.00026362079548206905, + "loss": 0.6527813720703125, + "mean_token_accuracy": 0.8083000521564004, + "num_tokens": 1393922.0, + "step": 450 + }, + { + "entropy": 0.6749556151032448, + "epoch": 1.1683226183518411, + "grad_norm": 0.6444424390792847, + "learning_rate": 0.0002634191889632654, + "loss": 0.627038345336914, + "mean_token_accuracy": 0.8141268998384475, + "num_tokens": 1551442.0, + "step": 500 + }, + { + "entropy": 0.6727319830656051, + "epoch": 1.2852133255406195, + "grad_norm": 0.6360110640525818, + "learning_rate": 0.00026299877604786883, + "loss": 0.6232248687744141, + "mean_token_accuracy": 0.8153417313098907, + "num_tokens": 1706697.0, + "step": 550 + }, + { + "entropy": 0.6600116823613643, + "epoch": 1.4021040327293979, + "grad_norm": 0.46475180983543396, + "learning_rate": 0.00026236025574535523, + "loss": 0.6124750518798828, + "mean_token_accuracy": 0.8175293403863907, + "num_tokens": 1865246.0, + "step": 600 + }, + { + "entropy": 0.6494863349199295, + "epoch": 1.5189947399181765, + "grad_norm": 0.6132755875587463, + "learning_rate": 0.00026150468970660587, + "loss": 0.5997534942626953, + "mean_token_accuracy": 0.822050162255764, + "num_tokens": 2022060.0, + "step": 650 + }, + { + "entropy": 0.6511707258224487, + "epoch": 1.635885447106955, + "grad_norm": 0.570785641670227, + "learning_rate": 0.00026043350045872797, + "loss": 0.6059062576293945, + "mean_token_accuracy": 0.820236160159111, + "num_tokens": 2178909.0, + "step": 700 + }, + { + "entropy": 0.6550412529706955, + "epoch": 1.7527761542957334, + "grad_norm": 0.5398300290107727, + "learning_rate": 0.0002591484690398568, + "loss": 0.6031087875366211, + "mean_token_accuracy": 0.8206016793847084, + "num_tokens": 2329314.0, + "step": 750 + }, + { + "entropy": 0.6509851542115211, + "epoch": 1.869666861484512, + "grad_norm": 0.6913411617279053, + "learning_rate": 0.0002576517320378704, + "loss": 0.6023480224609375, + "mean_token_accuracy": 0.822216115295887, + "num_tokens": 2478559.0, + "step": 800 + }, + { + "entropy": 0.6560930109024048, + "epoch": 1.9865575686732906, + "grad_norm": 0.5568020939826965, + "learning_rate": 0.0002559457780379408, + "loss": 0.6031641006469727, + "mean_token_accuracy": 0.8205807811021805, + "num_tokens": 2631937.0, + "step": 850 + }, + { + "epoch": 2.0, + "eval_entropy": 0.794637631866839, + "eval_loss": 0.7390629649162292, + "eval_mean_token_accuracy": 0.7983863295672776, + "eval_num_tokens": 2651286.0, + "eval_runtime": 36.7346, + "eval_samples_per_second": 33.511, + "eval_steps_per_second": 4.192, + "step": 856 + } + ], + "logging_steps": 50, + "max_steps": 4280, + "num_input_tokens_seen": 0, + "num_train_epochs": 10, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4.089312382391808e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..07feb095907d8ca70cf64cbd9167ace15b4e4f39 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-100/trainer_state.json @@ -0,0 +1,139 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.24554941682013506, + "eval_steps": 20, + "global_step": 100, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 4385149329799680.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e50a57dda9ec2ad3d24d3a78331251fbce7557fb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-120/trainer_state.json @@ -0,0 +1,160 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.2946593001841621, + "eval_steps": 20, + "global_step": 120, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 5253668222392320.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..306528192ea7d90fc62e2c168252db6d1c48eb61 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-140/trainer_state.json @@ -0,0 +1,181 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3437691835481891, + "eval_steps": 20, + "global_step": 140, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 6118471451159040.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..1dc6737bd60fe2bc3a1de00ffc91ebb503692c2a --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-160/trainer_state.json @@ -0,0 +1,202 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3928790669122161, + "eval_steps": 20, + "global_step": 160, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7013736112611840.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..f1b2721eda1078756a54e1e0c1d6caf763d318ac --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-180/trainer_state.json @@ -0,0 +1,223 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4419889502762431, + "eval_steps": 20, + "global_step": 180, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 7814847254853120.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..b2b6a96d0d8e21cecf76e6febd5c2d0685fb7a1b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-20/trainer_state.json @@ -0,0 +1,55 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.04910988336402701, + "eval_steps": 20, + "global_step": 20, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 882154677765120.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..15d8b96e6d87d3f41a68155a2c10fa8f33753798 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-200/trainer_state.json @@ -0,0 +1,244 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.4910988336402701, + "eval_steps": 20, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 8603198947729920.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..563306e129a331cecf8a3732127e315cfaf81ff9 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-220/trainer_state.json @@ -0,0 +1,265 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5402087170042971, + "eval_steps": 20, + "global_step": 220, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 9483916434769920.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c7a6bbf613b56a80bf6945be589d0ee01252bd42 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-240/trainer_state.json @@ -0,0 +1,286 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.5893186003683242, + "eval_steps": 20, + "global_step": 240, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.030707618594048e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad4c55943e70fd22f234252755c40e9227febdb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-260/trainer_state.json @@ -0,0 +1,307 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6384284837323512, + "eval_steps": 20, + "global_step": 260, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.12234079602176e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ac80833bab7fe02d8d676d814f403fb9d04c92d8 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-280/trainer_state.json @@ -0,0 +1,328 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6875383670963782, + "eval_steps": 20, + "global_step": 280, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.204183549802496e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..6f82990c2fbf5047472269cc86b324b35f1c571a --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-300/trainer_state.json @@ -0,0 +1,349 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7366482504604052, + "eval_steps": 20, + "global_step": 300, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.29431293498368e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9471bb5534d783834146a64df9bdbf4ea1bae2a4 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-320/trainer_state.json @@ -0,0 +1,370 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.7857581338244322, + "eval_steps": 20, + "global_step": 320, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.379271938897664e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8f1f23e179a1f64641a8fc0eb6653e18a4730eab --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-340/trainer_state.json @@ -0,0 +1,391 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8348680171884592, + "eval_steps": 20, + "global_step": 340, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.466169397618944e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..492d50334065285f77598321e19501a0f79f767e --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-360/trainer_state.json @@ -0,0 +1,412 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.8839779005524862, + "eval_steps": 20, + "global_step": 360, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.553852053224192e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..df52cc65a6a7b4bd7e732b12cd9289b0ebfdeae5 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-380/trainer_state.json @@ -0,0 +1,433 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9330877839165131, + "eval_steps": 20, + "global_step": 380, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.635372315427584e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8802e06314794d00a606546a836e779851ddd6bd --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-40/trainer_state.json @@ -0,0 +1,76 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.09821976672805402, + "eval_steps": 20, + "global_step": 40, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1795892498050560.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..4c36b7d64033266b75e70a3c631d490dc5fd3d6b --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-400/trainer_state.json @@ -0,0 +1,454 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9821976672805403, + "eval_steps": 20, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.724008424391936e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..54e42af62d5fe96454bcd84220b3cdb3f4c64013 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-420/trainer_state.json @@ -0,0 +1,475 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0294659300184161, + "eval_steps": 20, + "global_step": 420, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + }, + { + "entropy": 0.6647140879135627, + "epoch": 1.0294659300184161, + "grad_norm": 0.4148324429988861, + "learning_rate": 0.00012127523436919402, + "loss": 0.5996760845184326, + "mean_token_accuracy": 0.8205991890523341, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.6531744088445391, + "eval_loss": 0.6474375128746033, + "eval_mean_token_accuracy": 0.8118055742127555, + "eval_num_tokens": 1076273.0, + "eval_runtime": 57.7161, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 420 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.806675810168e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..563ff6e8594a33c55faa233ae21a7f8b45c59660 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-440/trainer_state.json @@ -0,0 +1,496 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.0785758133824432, + "eval_steps": 20, + "global_step": 440, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + }, + { + "entropy": 0.6647140879135627, + "epoch": 1.0294659300184161, + "grad_norm": 0.4148324429988861, + "learning_rate": 0.00012127523436919402, + "loss": 0.5996760845184326, + "mean_token_accuracy": 0.8205991890523341, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.6531744088445391, + "eval_loss": 0.6474375128746033, + "eval_mean_token_accuracy": 0.8118055742127555, + "eval_num_tokens": 1076273.0, + "eval_runtime": 57.7161, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 420 + }, + { + "entropy": 0.64856808334589, + "epoch": 1.0785758133824432, + "grad_norm": 0.4203054904937744, + "learning_rate": 0.0001121087738077257, + "loss": 0.580596923828125, + "mean_token_accuracy": 0.8229757599532604, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.6491447843824114, + "eval_loss": 0.6415509581565857, + "eval_mean_token_accuracy": 0.8132358625956944, + "eval_num_tokens": 1133367.0, + "eval_runtime": 57.0688, + "eval_samples_per_second": 24.462, + "eval_steps_per_second": 3.066, + "step": 440 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.899248419581888e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..fd518b30705bc32a44f7998f1590f93ba1593eb8 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-460/trainer_state.json @@ -0,0 +1,517 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1276856967464703, + "eval_steps": 20, + "global_step": 460, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + }, + { + "entropy": 0.6647140879135627, + "epoch": 1.0294659300184161, + "grad_norm": 0.4148324429988861, + "learning_rate": 0.00012127523436919402, + "loss": 0.5996760845184326, + "mean_token_accuracy": 0.8205991890523341, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.6531744088445391, + "eval_loss": 0.6474375128746033, + "eval_mean_token_accuracy": 0.8118055742127555, + "eval_num_tokens": 1076273.0, + "eval_runtime": 57.7161, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 420 + }, + { + "entropy": 0.64856808334589, + "epoch": 1.0785758133824432, + "grad_norm": 0.4203054904937744, + "learning_rate": 0.0001121087738077257, + "loss": 0.580596923828125, + "mean_token_accuracy": 0.8229757599532604, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.6491447843824114, + "eval_loss": 0.6415509581565857, + "eval_mean_token_accuracy": 0.8132358625956944, + "eval_num_tokens": 1133367.0, + "eval_runtime": 57.0688, + "eval_samples_per_second": 24.462, + "eval_steps_per_second": 3.066, + "step": 440 + }, + { + "entropy": 0.6387462258338928, + "epoch": 1.1276856967464703, + "grad_norm": 0.4547317326068878, + "learning_rate": 0.0001029086258676389, + "loss": 0.5705233097076416, + "mean_token_accuracy": 0.8258621700108051, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.653609961441585, + "eval_loss": 0.6332530975341797, + "eval_mean_token_accuracy": 0.8161680047852652, + "eval_num_tokens": 1186585.0, + "eval_runtime": 57.3472, + "eval_samples_per_second": 24.343, + "eval_steps_per_second": 3.052, + "step": 460 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.9901805283632e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..e5fd77ce1f980ae026dca5194704c78fa7fdd8f6 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-480/trainer_state.json @@ -0,0 +1,538 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1767955801104972, + "eval_steps": 20, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + }, + { + "entropy": 0.6647140879135627, + "epoch": 1.0294659300184161, + "grad_norm": 0.4148324429988861, + "learning_rate": 0.00012127523436919402, + "loss": 0.5996760845184326, + "mean_token_accuracy": 0.8205991890523341, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.6531744088445391, + "eval_loss": 0.6474375128746033, + "eval_mean_token_accuracy": 0.8118055742127555, + "eval_num_tokens": 1076273.0, + "eval_runtime": 57.7161, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 420 + }, + { + "entropy": 0.64856808334589, + "epoch": 1.0785758133824432, + "grad_norm": 0.4203054904937744, + "learning_rate": 0.0001121087738077257, + "loss": 0.580596923828125, + "mean_token_accuracy": 0.8229757599532604, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.6491447843824114, + "eval_loss": 0.6415509581565857, + "eval_mean_token_accuracy": 0.8132358625956944, + "eval_num_tokens": 1133367.0, + "eval_runtime": 57.0688, + "eval_samples_per_second": 24.462, + "eval_steps_per_second": 3.066, + "step": 440 + }, + { + "entropy": 0.6387462258338928, + "epoch": 1.1276856967464703, + "grad_norm": 0.4547317326068878, + "learning_rate": 0.0001029086258676389, + "loss": 0.5705233097076416, + "mean_token_accuracy": 0.8258621700108051, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.653609961441585, + "eval_loss": 0.6332530975341797, + "eval_mean_token_accuracy": 0.8161680047852652, + "eval_num_tokens": 1186585.0, + "eval_runtime": 57.3472, + "eval_samples_per_second": 24.343, + "eval_steps_per_second": 3.052, + "step": 460 + }, + { + "entropy": 0.6427005555480718, + "epoch": 1.1767955801104972, + "grad_norm": 0.4293302297592163, + "learning_rate": 9.374216530617052e-05, + "loss": 0.5639504432678223, + "mean_token_accuracy": 0.8271843157708645, + "num_tokens": 1235457.0, + "step": 480 + }, + { + "epoch": 1.1767955801104972, + "eval_entropy": 0.6493809008598328, + "eval_loss": 0.6308221817016602, + "eval_mean_token_accuracy": 0.8174078563281468, + "eval_num_tokens": 1235457.0, + "eval_runtime": 57.7158, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 480 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.07315200266464e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..227d8ad777734a57ddcc9813830b471fe12f4610 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-500/trainer_state.json @@ -0,0 +1,559 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2259054634745243, + "eval_steps": 20, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + }, + { + "entropy": 0.879152987897396, + "epoch": 0.24554941682013506, + "grad_norm": 1.067221760749817, + "learning_rate": 0.00021473293557345806, + "loss": 0.807522201538086, + "mean_token_accuracy": 0.772921597212553, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.8607830974033901, + "eval_loss": 0.8036007881164551, + "eval_mean_token_accuracy": 0.7755460752759661, + "eval_num_tokens": 253606.0, + "eval_runtime": 57.7576, + "eval_samples_per_second": 24.17, + "eval_steps_per_second": 3.03, + "step": 100 + }, + { + "entropy": 0.8589029781520366, + "epoch": 0.2946593001841621, + "grad_norm": 0.7390042543411255, + "learning_rate": 0.0002136721067543722, + "loss": 0.7829637050628662, + "mean_token_accuracy": 0.779672809690237, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8330987681661334, + "eval_loss": 0.7742031216621399, + "eval_mean_token_accuracy": 0.785119104044778, + "eval_num_tokens": 306583.0, + "eval_runtime": 57.3803, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 120 + }, + { + "entropy": 0.8094905942678452, + "epoch": 0.3437691835481891, + "grad_norm": 0.6055881381034851, + "learning_rate": 0.00021183381937537642, + "loss": 0.7396022796630859, + "mean_token_accuracy": 0.7867072127759457, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.7953300523757935, + "eval_loss": 0.7521983981132507, + "eval_mean_token_accuracy": 0.7905156462533134, + "eval_num_tokens": 361815.0, + "eval_runtime": 57.6345, + "eval_samples_per_second": 24.222, + "eval_steps_per_second": 3.036, + "step": 140 + }, + { + "entropy": 0.7955982603132725, + "epoch": 0.3928790669122161, + "grad_norm": 0.5252285599708557, + "learning_rate": 0.00020923153562932872, + "loss": 0.7219765186309814, + "mean_token_accuracy": 0.792796092480421, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.7623447510174343, + "eval_loss": 0.7298487424850464, + "eval_mean_token_accuracy": 0.7945844326700483, + "eval_num_tokens": 417429.0, + "eval_runtime": 57.9123, + "eval_samples_per_second": 24.105, + "eval_steps_per_second": 3.022, + "step": 160 + }, + { + "entropy": 0.7903043553233147, + "epoch": 0.4419889502762431, + "grad_norm": 0.6948235034942627, + "learning_rate": 0.00020588431262670093, + "loss": 0.7225021839141845, + "mean_token_accuracy": 0.791443546116352, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.790012172971453, + "eval_loss": 0.720366895198822, + "eval_mean_token_accuracy": 0.7943330574035644, + "eval_num_tokens": 467817.0, + "eval_runtime": 57.4578, + "eval_samples_per_second": 24.296, + "eval_steps_per_second": 3.046, + "step": 180 + }, + { + "entropy": 0.7773297265172004, + "epoch": 0.4910988336402701, + "grad_norm": 0.6160358786582947, + "learning_rate": 0.00020181666283606295, + "loss": 0.7100065231323243, + "mean_token_accuracy": 0.7956355385482311, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.783927013874054, + "eval_loss": 0.7202737927436829, + "eval_mean_token_accuracy": 0.7964657868657793, + "eval_num_tokens": 513773.0, + "eval_runtime": 57.6663, + "eval_samples_per_second": 24.208, + "eval_steps_per_second": 3.035, + "step": 200 + }, + { + "entropy": 0.7645334511995315, + "epoch": 0.5402087170042971, + "grad_norm": 0.6620849967002869, + "learning_rate": 0.00019705837457374996, + "loss": 0.7037524223327637, + "mean_token_accuracy": 0.7996554173529148, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.7768624458994184, + "eval_loss": 0.7051481604576111, + "eval_mean_token_accuracy": 0.7991346962111336, + "eval_num_tokens": 565606.0, + "eval_runtime": 57.7442, + "eval_samples_per_second": 24.176, + "eval_steps_per_second": 3.031, + "step": 220 + }, + { + "entropy": 0.7731568738818169, + "epoch": 0.5893186003683242, + "grad_norm": 0.553688645362854, + "learning_rate": 0.00019164429385730688, + "loss": 0.7003726959228516, + "mean_token_accuracy": 0.7977605648338795, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.7570131056649344, + "eval_loss": 0.7042225003242493, + "eval_mean_token_accuracy": 0.8000479991095407, + "eval_num_tokens": 616528.0, + "eval_runtime": 57.8713, + "eval_samples_per_second": 24.122, + "eval_steps_per_second": 3.024, + "step": 240 + }, + { + "entropy": 0.7343776792287826, + "epoch": 0.6384284837323512, + "grad_norm": 0.5576229095458984, + "learning_rate": 0.00018561406922024723, + "loss": 0.6735261440277099, + "mean_token_accuracy": 0.8059290319681167, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7231997234480722, + "eval_loss": 0.6926313638687134, + "eval_mean_token_accuracy": 0.8031776939119611, + "eval_num_tokens": 669440.0, + "eval_runtime": 57.1692, + "eval_samples_per_second": 24.419, + "eval_steps_per_second": 3.061, + "step": 260 + }, + { + "entropy": 0.7326226674020291, + "epoch": 0.6875383670963782, + "grad_norm": 0.5962154865264893, + "learning_rate": 0.00017901186135690458, + "loss": 0.6677989482879638, + "mean_token_accuracy": 0.8022022873163224, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7265675282478332, + "eval_loss": 0.6867054104804993, + "eval_mean_token_accuracy": 0.8021906747136798, + "eval_num_tokens": 720146.0, + "eval_runtime": 57.8067, + "eval_samples_per_second": 24.149, + "eval_steps_per_second": 3.027, + "step": 280 + }, + { + "entropy": 0.7409031428396702, + "epoch": 0.7366482504604052, + "grad_norm": 0.5716915726661682, + "learning_rate": 0.0001718860197237135, + "loss": 0.6692957878112793, + "mean_token_accuracy": 0.8036864519119262, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7272419779641288, + "eval_loss": 0.6771560907363892, + "eval_mean_token_accuracy": 0.8062130059514727, + "eval_num_tokens": 772624.0, + "eval_runtime": 57.5942, + "eval_samples_per_second": 24.239, + "eval_steps_per_second": 3.039, + "step": 300 + }, + { + "entropy": 0.7243741095066071, + "epoch": 0.7857581338244322, + "grad_norm": 0.5360459089279175, + "learning_rate": 0.00016428872846524118, + "loss": 0.6628021240234375, + "mean_token_accuracy": 0.8050650551915168, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7325100105149406, + "eval_loss": 0.6667791604995728, + "eval_mean_token_accuracy": 0.8082476602281843, + "eval_num_tokens": 822969.0, + "eval_runtime": 57.3807, + "eval_samples_per_second": 24.329, + "eval_steps_per_second": 3.05, + "step": 320 + }, + { + "entropy": 0.7213595665991306, + "epoch": 0.8348680171884592, + "grad_norm": 0.48658841848373413, + "learning_rate": 0.00015627562425793356, + "loss": 0.6494232654571533, + "mean_token_accuracy": 0.811632139235735, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7198746153286525, + "eval_loss": 0.6634471416473389, + "eval_mean_token_accuracy": 0.8094694914136614, + "eval_num_tokens": 873459.0, + "eval_runtime": 57.6988, + "eval_samples_per_second": 24.195, + "eval_steps_per_second": 3.033, + "step": 340 + }, + { + "entropy": 0.7133754774928093, + "epoch": 0.8839779005524862, + "grad_norm": 0.685793399810791, + "learning_rate": 0.00014790538887019238, + "loss": 0.642530870437622, + "mean_token_accuracy": 0.8122274152934551, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.6683637884684971, + "eval_loss": 0.6579923033714294, + "eval_mean_token_accuracy": 0.811241352558136, + "eval_num_tokens": 925669.0, + "eval_runtime": 57.8503, + "eval_samples_per_second": 24.131, + "eval_steps_per_second": 3.025, + "step": 360 + }, + { + "entropy": 0.7262612670660019, + "epoch": 0.9330877839165131, + "grad_norm": 0.6348966956138611, + "learning_rate": 0.00013923931942255727, + "loss": 0.656022596359253, + "mean_token_accuracy": 0.8072879828512669, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7012172797748021, + "eval_loss": 0.651187002658844, + "eval_mean_token_accuracy": 0.8117410482679095, + "eval_num_tokens": 973436.0, + "eval_runtime": 57.1997, + "eval_samples_per_second": 24.406, + "eval_steps_per_second": 3.059, + "step": 380 + }, + { + "entropy": 0.7007582865655422, + "epoch": 0.9821976672805403, + "grad_norm": 0.4913424849510193, + "learning_rate": 0.0001303408794950743, + "loss": 0.6367915153503418, + "mean_token_accuracy": 0.8122764162719249, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.6719393570082528, + "eval_loss": 0.6448967456817627, + "eval_mean_token_accuracy": 0.8140764743941171, + "eval_num_tokens": 1026370.0, + "eval_runtime": 57.9405, + "eval_samples_per_second": 24.094, + "eval_steps_per_second": 3.02, + "step": 400 + }, + { + "entropy": 0.6647140879135627, + "epoch": 1.0294659300184161, + "grad_norm": 0.4148324429988861, + "learning_rate": 0.00012127523436919402, + "loss": 0.5996760845184326, + "mean_token_accuracy": 0.8205991890523341, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.6531744088445391, + "eval_loss": 0.6474375128746033, + "eval_mean_token_accuracy": 0.8118055742127555, + "eval_num_tokens": 1076273.0, + "eval_runtime": 57.7161, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 420 + }, + { + "entropy": 0.64856808334589, + "epoch": 1.0785758133824432, + "grad_norm": 0.4203054904937744, + "learning_rate": 0.0001121087738077257, + "loss": 0.580596923828125, + "mean_token_accuracy": 0.8229757599532604, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.6491447843824114, + "eval_loss": 0.6415509581565857, + "eval_mean_token_accuracy": 0.8132358625956944, + "eval_num_tokens": 1133367.0, + "eval_runtime": 57.0688, + "eval_samples_per_second": 24.462, + "eval_steps_per_second": 3.066, + "step": 440 + }, + { + "entropy": 0.6387462258338928, + "epoch": 1.1276856967464703, + "grad_norm": 0.4547317326068878, + "learning_rate": 0.0001029086258676389, + "loss": 0.5705233097076416, + "mean_token_accuracy": 0.8258621700108051, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.653609961441585, + "eval_loss": 0.6332530975341797, + "eval_mean_token_accuracy": 0.8161680047852652, + "eval_num_tokens": 1186585.0, + "eval_runtime": 57.3472, + "eval_samples_per_second": 24.343, + "eval_steps_per_second": 3.052, + "step": 460 + }, + { + "entropy": 0.6427005555480718, + "epoch": 1.1767955801104972, + "grad_norm": 0.4293302297592163, + "learning_rate": 9.374216530617052e-05, + "loss": 0.5639504432678223, + "mean_token_accuracy": 0.8271843157708645, + "num_tokens": 1235457.0, + "step": 480 + }, + { + "epoch": 1.1767955801104972, + "eval_entropy": 0.6493809008598328, + "eval_loss": 0.6308221817016602, + "eval_mean_token_accuracy": 0.8174078563281468, + "eval_num_tokens": 1235457.0, + "eval_runtime": 57.7158, + "eval_samples_per_second": 24.187, + "eval_steps_per_second": 3.032, + "step": 480 + }, + { + "entropy": 0.617260966822505, + "epoch": 1.2259054634745243, + "grad_norm": 0.44997501373291016, + "learning_rate": 8.46765201802903e-05, + "loss": 0.5545801639556884, + "mean_token_accuracy": 0.8310177803039551, + "num_tokens": 1289995.0, + "step": 500 + }, + { + "epoch": 1.2259054634745243, + "eval_entropy": 0.6463884047099522, + "eval_loss": 0.6248639822006226, + "eval_mean_token_accuracy": 0.8196247465269906, + "eval_num_tokens": 1289995.0, + "eval_runtime": 57.655, + "eval_samples_per_second": 24.213, + "eval_steps_per_second": 3.035, + "step": 500 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.162562792483264e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..49f099052acaaa62e5a9a6914ef6590be09f26d9 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-60/trainer_state.json @@ -0,0 +1,97 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.14732965009208104, + "eval_steps": 20, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2661782383219200.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9231547eca790e866e907eade83a12f382230c57 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 128, + "lora_bias": false, + "lora_dropout": 0.056708081859357024, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 128, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ccc6fc777cca7152d025ec97b396eae99ecce8e2 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test1/checkpoint-80/trainer_state.json @@ -0,0 +1,118 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19643953345610804, + "eval_steps": 20, + "global_step": 80, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.136037287116051, + "epoch": 0.04910988336402701, + "grad_norm": 1.5031570196151733, + "learning_rate": 4.982110480282838e-05, + "loss": 1.9729637145996093, + "mean_token_accuracy": 0.5955793052911759, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 1.2274160248892647, + "eval_loss": 1.211025595664978, + "eval_mean_token_accuracy": 0.6963188947950091, + "eval_num_tokens": 51407.0, + "eval_runtime": 58.8096, + "eval_samples_per_second": 23.738, + "eval_steps_per_second": 2.976, + "step": 20 + }, + { + "entropy": 1.0958118669688701, + "epoch": 0.09821976672805402, + "grad_norm": 1.0549366474151611, + "learning_rate": 0.00010226437301633193, + "loss": 1.0308157920837402, + "mean_token_accuracy": 0.7261112071573734, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 0.9666143652370998, + "eval_loss": 0.9403602480888367, + "eval_mean_token_accuracy": 0.7466924602644784, + "eval_num_tokens": 107414.0, + "eval_runtime": 57.3655, + "eval_samples_per_second": 24.335, + "eval_steps_per_second": 3.051, + "step": 40 + }, + { + "entropy": 0.9621845617890358, + "epoch": 0.14732965009208104, + "grad_norm": 0.8900883793830872, + "learning_rate": 0.00015470764122983547, + "loss": 0.8886768341064453, + "mean_token_accuracy": 0.7551977805793285, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 0.9346491888591222, + "eval_loss": 0.8509624600410461, + "eval_mean_token_accuracy": 0.7676705942835127, + "eval_num_tokens": 157071.0, + "eval_runtime": 57.519, + "eval_samples_per_second": 24.27, + "eval_steps_per_second": 3.042, + "step": 60 + }, + { + "entropy": 0.9030815377831459, + "epoch": 0.19643953345610804, + "grad_norm": 0.9833227396011353, + "learning_rate": 0.00020715090944333904, + "loss": 0.8231673240661621, + "mean_token_accuracy": 0.7664932057261467, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.8813995848383223, + "eval_loss": 0.8210445642471313, + "eval_mean_token_accuracy": 0.7710604834556579, + "eval_num_tokens": 204521.0, + "eval_runtime": 57.755, + "eval_samples_per_second": 24.171, + "eval_steps_per_second": 3.03, + "step": 80 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3555048999029760.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..9b9a4dda9e8e58ad1514a0a6387528f54a2c30c5 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-480/trainer_state.json @@ -0,0 +1,538 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.1767955801104972, + "eval_steps": 20, + "global_step": 480, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + }, + { + "entropy": 0.7288912821125675, + "epoch": 1.0294659300184161, + "grad_norm": 0.9075890779495239, + "learning_rate": 4.729208140100435e-05, + "loss": 0.6533905506134033, + "mean_token_accuracy": 0.807745551908171, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.7279435137340001, + "eval_loss": 0.6758045554161072, + "eval_mean_token_accuracy": 0.8043134975433349, + "eval_num_tokens": 1076273.0, + "eval_runtime": 53.8173, + "eval_samples_per_second": 25.921, + "eval_steps_per_second": 3.252, + "step": 420 + }, + { + "entropy": 0.7183651916682721, + "epoch": 1.0785758133824432, + "grad_norm": 0.9271263480186462, + "learning_rate": 4.371755935380415e-05, + "loss": 0.6382484912872315, + "mean_token_accuracy": 0.810582410544157, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.7224361307280405, + "eval_loss": 0.6700165271759033, + "eval_mean_token_accuracy": 0.8052176751409258, + "eval_num_tokens": 1133367.0, + "eval_runtime": 54.2346, + "eval_samples_per_second": 25.722, + "eval_steps_per_second": 3.227, + "step": 440 + }, + { + "entropy": 0.7097452029585838, + "epoch": 1.1276856967464703, + "grad_norm": 0.8726329207420349, + "learning_rate": 4.012990069004659e-05, + "loss": 0.6309147834777832, + "mean_token_accuracy": 0.8141021691262722, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.7231041298593793, + "eval_loss": 0.6672903299331665, + "eval_mean_token_accuracy": 0.8079326885087149, + "eval_num_tokens": 1186585.0, + "eval_runtime": 53.7923, + "eval_samples_per_second": 25.933, + "eval_steps_per_second": 3.253, + "step": 460 + }, + { + "entropy": 0.7170382678508759, + "epoch": 1.1767955801104972, + "grad_norm": 0.8970508575439453, + "learning_rate": 3.6555378642846364e-05, + "loss": 0.6294277667999267, + "mean_token_accuracy": 0.8117733597755432, + "num_tokens": 1235457.0, + "step": 480 + }, + { + "epoch": 1.1767955801104972, + "eval_entropy": 0.7064940530913216, + "eval_loss": 0.6625319123268127, + "eval_mean_token_accuracy": 0.8093922696794782, + "eval_num_tokens": 1235457.0, + "eval_runtime": 53.6574, + "eval_samples_per_second": 25.998, + "eval_steps_per_second": 3.261, + "step": 480 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 1.980221720242368e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..c6f3eac6c0aaf9262862579223f8aa385cf431fe --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-500/trainer_state.json @@ -0,0 +1,559 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.2259054634745243, + "eval_steps": 20, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + }, + { + "entropy": 0.9607135757803917, + "epoch": 0.24554941682013506, + "grad_norm": 2.213541030883789, + "learning_rate": 8.373653138200974e-05, + "loss": 0.8742897987365723, + "mean_token_accuracy": 0.7584197156131267, + "num_tokens": 253606.0, + "step": 100 + }, + { + "epoch": 0.24554941682013506, + "eval_entropy": 0.9265402409008571, + "eval_loss": 0.8480852246284485, + "eval_mean_token_accuracy": 0.7668960053580148, + "eval_num_tokens": 253606.0, + "eval_runtime": 54.5404, + "eval_samples_per_second": 25.577, + "eval_steps_per_second": 3.209, + "step": 100 + }, + { + "entropy": 0.9136901050806046, + "epoch": 0.2946593001841621, + "grad_norm": 1.2657700777053833, + "learning_rate": 8.332285415330192e-05, + "loss": 0.8283636093139648, + "mean_token_accuracy": 0.7690222583711147, + "num_tokens": 306583.0, + "step": 120 + }, + { + "epoch": 0.2946593001841621, + "eval_entropy": 0.8823672638620649, + "eval_loss": 0.8114516139030457, + "eval_mean_token_accuracy": 0.7755525919369289, + "eval_num_tokens": 306583.0, + "eval_runtime": 55.3013, + "eval_samples_per_second": 25.225, + "eval_steps_per_second": 3.164, + "step": 120 + }, + { + "entropy": 0.8614003121852875, + "epoch": 0.3437691835481891, + "grad_norm": 1.0693161487579346, + "learning_rate": 8.260600180650497e-05, + "loss": 0.778686809539795, + "mean_token_accuracy": 0.7784358590841294, + "num_tokens": 361815.0, + "step": 140 + }, + { + "epoch": 0.3437691835481891, + "eval_entropy": 0.8277448885781424, + "eval_loss": 0.7953144311904907, + "eval_mean_token_accuracy": 0.7796006223133632, + "eval_num_tokens": 361815.0, + "eval_runtime": 55.4206, + "eval_samples_per_second": 25.171, + "eval_steps_per_second": 3.158, + "step": 140 + }, + { + "entropy": 0.8520236104726792, + "epoch": 0.3928790669122161, + "grad_norm": 1.038674235343933, + "learning_rate": 8.159122401294533e-05, + "loss": 0.7658556938171387, + "mean_token_accuracy": 0.7823643557727337, + "num_tokens": 417429.0, + "step": 160 + }, + { + "epoch": 0.3928790669122161, + "eval_entropy": 0.8204359361103603, + "eval_loss": 0.7691255807876587, + "eval_mean_token_accuracy": 0.7845319850104195, + "eval_num_tokens": 417429.0, + "eval_runtime": 55.1624, + "eval_samples_per_second": 25.289, + "eval_steps_per_second": 3.172, + "step": 160 + }, + { + "entropy": 0.8366772942245007, + "epoch": 0.4419889502762431, + "grad_norm": 1.441671371459961, + "learning_rate": 8.028595221915362e-05, + "loss": 0.7603907108306884, + "mean_token_accuracy": 0.7837655946612359, + "num_tokens": 467817.0, + "step": 180 + }, + { + "epoch": 0.4419889502762431, + "eval_entropy": 0.8421429433141436, + "eval_loss": 0.7569069862365723, + "eval_mean_token_accuracy": 0.7865874515260969, + "eval_num_tokens": 467817.0, + "eval_runtime": 55.1833, + "eval_samples_per_second": 25.279, + "eval_steps_per_second": 3.171, + "step": 180 + }, + { + "entropy": 0.8282789587974548, + "epoch": 0.4910988336402701, + "grad_norm": 1.1421457529067993, + "learning_rate": 7.86997452247065e-05, + "loss": 0.7440836906433106, + "mean_token_accuracy": 0.7888234727084636, + "num_tokens": 513773.0, + "step": 200 + }, + { + "epoch": 0.4910988336402701, + "eval_entropy": 0.8229101082256862, + "eval_loss": 0.7469918131828308, + "eval_mean_token_accuracy": 0.7869657233783177, + "eval_num_tokens": 513773.0, + "eval_runtime": 54.7451, + "eval_samples_per_second": 25.482, + "eval_steps_per_second": 3.197, + "step": 200 + }, + { + "entropy": 0.8064627222716808, + "epoch": 0.5402087170042971, + "grad_norm": 1.1229082345962524, + "learning_rate": 7.684421918098267e-05, + "loss": 0.732261323928833, + "mean_token_accuracy": 0.7928754203021526, + "num_tokens": 565606.0, + "step": 220 + }, + { + "epoch": 0.5402087170042971, + "eval_entropy": 0.8189267148290361, + "eval_loss": 0.7362269759178162, + "eval_mean_token_accuracy": 0.7914395029204232, + "eval_num_tokens": 565606.0, + "eval_runtime": 54.9856, + "eval_samples_per_second": 25.37, + "eval_steps_per_second": 3.183, + "step": 220 + }, + { + "entropy": 0.8221734672784805, + "epoch": 0.5893186003683242, + "grad_norm": 1.1489790678024292, + "learning_rate": 7.473296252346783e-05, + "loss": 0.7320152282714844, + "mean_token_accuracy": 0.789833651483059, + "num_tokens": 616528.0, + "step": 240 + }, + { + "epoch": 0.5893186003683242, + "eval_entropy": 0.8049671122006008, + "eval_loss": 0.728659987449646, + "eval_mean_token_accuracy": 0.7936891293525696, + "eval_num_tokens": 616528.0, + "eval_runtime": 54.285, + "eval_samples_per_second": 25.698, + "eval_steps_per_second": 3.224, + "step": 240 + }, + { + "entropy": 0.7805217795073986, + "epoch": 0.6384284837323512, + "grad_norm": 0.9496471285820007, + "learning_rate": 7.238143646057855e-05, + "loss": 0.7050290584564209, + "mean_token_accuracy": 0.7986594937741757, + "num_tokens": 669440.0, + "step": 260 + }, + { + "epoch": 0.6384284837323512, + "eval_entropy": 0.7794270893505642, + "eval_loss": 0.7148454785346985, + "eval_mean_token_accuracy": 0.7973272456441607, + "eval_num_tokens": 669440.0, + "eval_runtime": 53.9884, + "eval_samples_per_second": 25.839, + "eval_steps_per_second": 3.241, + "step": 260 + }, + { + "entropy": 0.781412948668003, + "epoch": 0.6875383670963782, + "grad_norm": 1.1252366304397583, + "learning_rate": 6.980686174774777e-05, + "loss": 0.6978650093078613, + "mean_token_accuracy": 0.7952493160963059, + "num_tokens": 720146.0, + "step": 280 + }, + { + "epoch": 0.6875383670963782, + "eval_entropy": 0.7790967045511519, + "eval_loss": 0.7110744118690491, + "eval_mean_token_accuracy": 0.7964006056104388, + "eval_num_tokens": 720146.0, + "eval_runtime": 53.9404, + "eval_samples_per_second": 25.862, + "eval_steps_per_second": 3.244, + "step": 280 + }, + { + "entropy": 0.7788082852959632, + "epoch": 0.7366482504604052, + "grad_norm": 1.1519086360931396, + "learning_rate": 6.702809257595105e-05, + "loss": 0.7019347190856934, + "mean_token_accuracy": 0.7960198536515236, + "num_tokens": 772624.0, + "step": 300 + }, + { + "epoch": 0.7366482504604052, + "eval_entropy": 0.7932151372092111, + "eval_loss": 0.7035816311836243, + "eval_mean_token_accuracy": 0.7979611914498466, + "eval_num_tokens": 772624.0, + "eval_runtime": 54.0517, + "eval_samples_per_second": 25.809, + "eval_steps_per_second": 3.238, + "step": 300 + }, + { + "entropy": 0.7729029789566993, + "epoch": 0.7857581338244322, + "grad_norm": 1.0247046947479248, + "learning_rate": 6.406547849821584e-05, + "loss": 0.6924194812774658, + "mean_token_accuracy": 0.7985413707792759, + "num_tokens": 822969.0, + "step": 320 + }, + { + "epoch": 0.7857581338244322, + "eval_entropy": 0.7812128639221192, + "eval_loss": 0.6989337205886841, + "eval_mean_token_accuracy": 0.8005841762678964, + "eval_num_tokens": 822969.0, + "eval_runtime": 54.1306, + "eval_samples_per_second": 25.771, + "eval_steps_per_second": 3.233, + "step": 320 + }, + { + "entropy": 0.7627703189849854, + "epoch": 0.8348680171884592, + "grad_norm": 0.957506537437439, + "learning_rate": 6.0940715405257566e-05, + "loss": 0.679268217086792, + "mean_token_accuracy": 0.8040587857365609, + "num_tokens": 873459.0, + "step": 340 + }, + { + "epoch": 0.8348680171884592, + "eval_entropy": 0.7624005011149815, + "eval_loss": 0.6934819221496582, + "eval_mean_token_accuracy": 0.8014607231957572, + "eval_num_tokens": 873459.0, + "eval_runtime": 54.1656, + "eval_samples_per_second": 25.754, + "eval_steps_per_second": 3.231, + "step": 340 + }, + { + "entropy": 0.763699171692133, + "epoch": 0.8839779005524862, + "grad_norm": 1.4329766035079956, + "learning_rate": 5.767668664158135e-05, + "loss": 0.6764235019683837, + "mean_token_accuracy": 0.8051542706787587, + "num_tokens": 925669.0, + "step": 360 + }, + { + "epoch": 0.8839779005524862, + "eval_entropy": 0.7163350166593279, + "eval_loss": 0.6882685422897339, + "eval_mean_token_accuracy": 0.802961312021528, + "eval_num_tokens": 925669.0, + "eval_runtime": 53.8288, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 360 + }, + { + "entropy": 0.7710359543561935, + "epoch": 0.9330877839165131, + "grad_norm": 1.141253113746643, + "learning_rate": 5.429729542559189e-05, + "loss": 0.6897628784179688, + "mean_token_accuracy": 0.8005676351487636, + "num_tokens": 973436.0, + "step": 380 + }, + { + "epoch": 0.9330877839165131, + "eval_entropy": 0.7462973778588431, + "eval_loss": 0.6829195022583008, + "eval_mean_token_accuracy": 0.8036447630609784, + "eval_num_tokens": 973436.0, + "eval_runtime": 53.8276, + "eval_samples_per_second": 25.916, + "eval_steps_per_second": 3.251, + "step": 380 + }, + { + "entropy": 0.7500453963875771, + "epoch": 0.9821976672805403, + "grad_norm": 0.9055600166320801, + "learning_rate": 5.0827289800936774e-05, + "loss": 0.6726306915283203, + "mean_token_accuracy": 0.8026990614831447, + "num_tokens": 1026370.0, + "step": 400 + }, + { + "epoch": 0.9821976672805403, + "eval_entropy": 0.745090776511601, + "eval_loss": 0.6795706152915955, + "eval_mean_token_accuracy": 0.8038556255613054, + "eval_num_tokens": 1026370.0, + "eval_runtime": 53.9485, + "eval_samples_per_second": 25.858, + "eval_steps_per_second": 3.244, + "step": 400 + }, + { + "entropy": 0.7288912821125675, + "epoch": 1.0294659300184161, + "grad_norm": 0.9075890779495239, + "learning_rate": 4.729208140100435e-05, + "loss": 0.6533905506134033, + "mean_token_accuracy": 0.807745551908171, + "num_tokens": 1076273.0, + "step": 420 + }, + { + "epoch": 1.0294659300184161, + "eval_entropy": 0.7279435137340001, + "eval_loss": 0.6758045554161072, + "eval_mean_token_accuracy": 0.8043134975433349, + "eval_num_tokens": 1076273.0, + "eval_runtime": 53.8173, + "eval_samples_per_second": 25.921, + "eval_steps_per_second": 3.252, + "step": 420 + }, + { + "entropy": 0.7183651916682721, + "epoch": 1.0785758133824432, + "grad_norm": 0.9271263480186462, + "learning_rate": 4.371755935380415e-05, + "loss": 0.6382484912872315, + "mean_token_accuracy": 0.810582410544157, + "num_tokens": 1133367.0, + "step": 440 + }, + { + "epoch": 1.0785758133824432, + "eval_entropy": 0.7224361307280405, + "eval_loss": 0.6700165271759033, + "eval_mean_token_accuracy": 0.8052176751409258, + "eval_num_tokens": 1133367.0, + "eval_runtime": 54.2346, + "eval_samples_per_second": 25.722, + "eval_steps_per_second": 3.227, + "step": 440 + }, + { + "entropy": 0.7097452029585838, + "epoch": 1.1276856967464703, + "grad_norm": 0.8726329207420349, + "learning_rate": 4.012990069004659e-05, + "loss": 0.6309147834777832, + "mean_token_accuracy": 0.8141021691262722, + "num_tokens": 1186585.0, + "step": 460 + }, + { + "epoch": 1.1276856967464703, + "eval_entropy": 0.7231041298593793, + "eval_loss": 0.6672903299331665, + "eval_mean_token_accuracy": 0.8079326885087149, + "eval_num_tokens": 1186585.0, + "eval_runtime": 53.7923, + "eval_samples_per_second": 25.933, + "eval_steps_per_second": 3.253, + "step": 460 + }, + { + "entropy": 0.7170382678508759, + "epoch": 1.1767955801104972, + "grad_norm": 0.8970508575439453, + "learning_rate": 3.6555378642846364e-05, + "loss": 0.6294277667999267, + "mean_token_accuracy": 0.8117733597755432, + "num_tokens": 1235457.0, + "step": 480 + }, + { + "epoch": 1.1767955801104972, + "eval_entropy": 0.7064940530913216, + "eval_loss": 0.6625319123268127, + "eval_mean_token_accuracy": 0.8093922696794782, + "eval_num_tokens": 1235457.0, + "eval_runtime": 53.6574, + "eval_samples_per_second": 25.998, + "eval_steps_per_second": 3.261, + "step": 480 + }, + { + "entropy": 0.6913388505578041, + "epoch": 1.2259054634745243, + "grad_norm": 0.9702943563461304, + "learning_rate": 3.3020170242913956e-05, + "loss": 0.616893196105957, + "mean_token_accuracy": 0.8174013815820217, + "num_tokens": 1289995.0, + "step": 500 + }, + { + "epoch": 1.2259054634745243, + "eval_entropy": 0.7072116715567452, + "eval_loss": 0.6600467562675476, + "eval_mean_token_accuracy": 0.8099266011374338, + "eval_num_tokens": 1289995.0, + "eval_runtime": 53.6227, + "eval_samples_per_second": 26.015, + "eval_steps_per_second": 3.264, + "step": 500 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2.065624617760397e+16, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..3a92660e6fafc54b52681103e85e54bc17d04c26 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-60/trainer_state.json @@ -0,0 +1,97 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.14732965009208104, + "eval_steps": 20, + "global_step": 60, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 2542466390807040.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7ac50e2706e5b4eae8f28e204601afb241c49e55 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/README.md @@ -0,0 +1,209 @@ +--- +base_model: Qwen/Qwen3.5-2B-Base +library_name: peft +pipeline_tag: text-generation +tags: +- base_model:adapter:Qwen/Qwen3.5-2B-Base +- lora +- sft +- transformers +- trl +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.19.1 \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..6d05047585cef4bba4ea582a1f394fe467801e35 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/adapter_config.json @@ -0,0 +1,48 @@ +{ + "alora_invocation_tokens": null, + "alpha_pattern": {}, + "arrow_config": null, + "auto_mapping": null, + "base_model_name_or_path": "Qwen/Qwen3.5-2B-Base", + "bias": "none", + "corda_config": null, + "ensure_weight_tying": false, + "eva_config": null, + "exclude_modules": null, + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 64, + "lora_bias": false, + "lora_dropout": 0.055708383706168485, + "lora_ga_config": null, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "peft_version": "0.19.1", + "qalora_group_size": 16, + "r": 32, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "k_proj", + "gate_proj", + "v_proj", + "down_proj", + "o_proj", + "up_proj", + "q_proj" + ], + "target_parameters": null, + "task_type": "CAUSAL_LM", + "trainable_token_indices": null, + "use_bdlora": null, + "use_dora": false, + "use_qalora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..0ef09f214eaa6d9bca297988afc1454b5827b2c7 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/chat_template.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- raise_exception('System message must be at the beginning.') }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if loop.index0 > ns.last_query_index %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is true %} + {{- '\n' }} + {%- else %} + {{- '\n\n\n\n' }} + {%- endif %} +{%- endif %} \ No newline at end of file diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..b4a37b2a6fd3ab3317cd7bac72855be1a843b2bb --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/tokenizer_config.json @@ -0,0 +1,31 @@ +{ + "add_prefix_space": false, + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "backend": "tokenizers", + "bos_token": null, + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "errors": "replace", + "image_token": "<|image_pad|>", + "is_local": false, + "model_max_length": 262144, + "model_specific_special_tokens": { + "audio_bos_token": "<|audio_start|>", + "audio_eos_token": "<|audio_end|>", + "audio_token": "<|audio_pad|>", + "image_token": "<|image_pad|>", + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" + }, + "pad_token": "<|endoftext|>", + "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", + "split_special_tokens": false, + "tokenizer_class": "TokenizersBackend", + "unk_token": null, + "video_token": "<|video_pad|>", + "vision_bos_token": "<|vision_start|>", + "vision_eos_token": "<|vision_end|>" +} diff --git a/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..497471f06721a8c4776e66b7db00f99b473ef627 --- /dev/null +++ b/overgeneralisation_original_Estonian/Qwen3.5-2B-Base_overgeneralisation_splits_original_features_train_overgeneralisation_splits_original_features_test2/checkpoint-80/trainer_state.json @@ -0,0 +1,118 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.19643953345610804, + "eval_steps": 20, + "global_step": 80, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "entropy": 2.5655054792761804, + "epoch": 0.04910988336402701, + "grad_norm": 2.106731653213501, + "learning_rate": 1.9428070010160535e-05, + "loss": 2.3814184188842775, + "mean_token_accuracy": 0.5490218658000231, + "num_tokens": 51407.0, + "step": 20 + }, + { + "epoch": 0.04910988336402701, + "eval_entropy": 2.196743746485029, + "eval_loss": 2.028722047805786, + "eval_mean_token_accuracy": 0.5899124583176204, + "eval_num_tokens": 51407.0, + "eval_runtime": 55.8819, + "eval_samples_per_second": 24.963, + "eval_steps_per_second": 3.132, + "step": 20 + }, + { + "entropy": 1.6433217093348502, + "epoch": 0.09821976672805402, + "grad_norm": 1.7636586427688599, + "learning_rate": 3.987867002085583e-05, + "loss": 1.4883951187133788, + "mean_token_accuracy": 0.6520100250840187, + "num_tokens": 107414.0, + "step": 40 + }, + { + "epoch": 0.09821976672805402, + "eval_entropy": 1.2071456554957798, + "eval_loss": 1.1546095609664917, + "eval_mean_token_accuracy": 0.7017311205182757, + "eval_num_tokens": 107414.0, + "eval_runtime": 55.21, + "eval_samples_per_second": 25.267, + "eval_steps_per_second": 3.17, + "step": 40 + }, + { + "entropy": 1.1379728086292744, + "epoch": 0.14732965009208104, + "grad_norm": 1.6891512870788574, + "learning_rate": 6.032927003155113e-05, + "loss": 1.0574092864990234, + "mean_token_accuracy": 0.7190469801425934, + "num_tokens": 157071.0, + "step": 60 + }, + { + "epoch": 0.14732965009208104, + "eval_entropy": 1.026683736869267, + "eval_loss": 0.9847686290740967, + "eval_mean_token_accuracy": 0.7383982699257987, + "eval_num_tokens": 157071.0, + "eval_runtime": 54.8863, + "eval_samples_per_second": 25.416, + "eval_steps_per_second": 3.188, + "step": 60 + }, + { + "entropy": 1.0109580919146537, + "epoch": 0.19643953345610804, + "grad_norm": 1.7396622896194458, + "learning_rate": 8.077987004224643e-05, + "loss": 0.9192052841186523, + "mean_token_accuracy": 0.7471949681639671, + "num_tokens": 204521.0, + "step": 80 + }, + { + "epoch": 0.19643953345610804, + "eval_entropy": 0.9901534714017596, + "eval_loss": 0.8948126435279846, + "eval_mean_token_accuracy": 0.7570484624590192, + "eval_num_tokens": 204521.0, + "eval_runtime": 54.924, + "eval_samples_per_second": 25.399, + "eval_steps_per_second": 3.186, + "step": 80 + } + ], + "logging_steps": 20, + "max_steps": 816, + "num_input_tokens_seen": 0, + "num_train_epochs": 2, + "save_steps": 20, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 3395691794598912.0, + "train_batch_size": 4, + "trial_name": null, + "trial_params": null +}