Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,75 @@
+---
+library_name: transformers
+license: other
+base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: code_finetune
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# code_finetune
+
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B) on the open_thoughts_19K dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.5804
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 2
+- total_train_batch_size: 2
+- total_eval_batch_size: 2
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1.0
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| No log        | 0.0870 | 2    | 0.6561          |
+| No log        | 0.1739 | 4    | 0.6379          |
+| No log        | 0.2609 | 6    | 0.6213          |
+| No log        | 0.3478 | 8    | 0.6110          |
+| No log        | 0.4348 | 10   | 0.5980          |
+| No log        | 0.5217 | 12   | 0.5886          |
+| No log        | 0.6087 | 14   | 0.5854          |
+| No log        | 0.6957 | 16   | 0.5828          |
+| No log        | 0.7826 | 18   | 0.5815          |
+| No log        | 0.8696 | 20   | 0.5806          |
+| No log        | 0.9565 | 22   | 0.5804          |
+
+
+### Framework versions
+
+- Transformers 4.50.0
+- Pytorch 2.6.0+cu124
+- Datasets 3.3.2
+- Tokenizers 0.21.0

all_results.json

@@ -0,0 +1,12 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.5803781151771545,
+    "eval_runtime": 1.3905,
+    "eval_samples_per_second": 3.596,
+    "eval_steps_per_second": 2.157,
+    "total_flos": 331406114816.0,
+    "train_loss": 0.629167598226796,
+    "train_runtime": 86.9566,
+    "train_samples_per_second": 0.517,
+    "train_steps_per_second": 0.264
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.50.0",
+  "use_cache": false,
+  "use_mrope": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

eval_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.5803781151771545,
+    "eval_runtime": 1.3905,
+    "eval_samples_per_second": 3.596,
+    "eval_steps_per_second": 2.157
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 151646,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "temperature": 0.6,
+  "top_p": 0.95,
+  "transformers_version": "4.50.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7aebde4368442e530cba1e43879681b0a401e33401275f5fd1f33bcb37d5b8e9
+size 3554214752

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893
+size 11422778

tokenizer_config.json

@@ -0,0 +1,197 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<｜User｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151645": {
+      "content": "<｜Assistant｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151646": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|EOT|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151648": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151649": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin��>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 16384,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 331406114816.0,
+    "train_loss": 0.629167598226796,
+    "train_runtime": 86.9566,
+    "train_samples_per_second": 0.517,
+    "train_steps_per_second": 0.264
+}

trainer_log.jsonl

@@ -0,0 +1,12 @@
+{"current_steps": 2, "total_steps": 23, "eval_loss": 0.6561041474342346, "epoch": 0.08695652173913043, "percentage": 8.7, "elapsed_time": "0:00:05", "remaining_time": "0:00:54"}
+{"current_steps": 4, "total_steps": 23, "eval_loss": 0.6379454731941223, "epoch": 0.17391304347826086, "percentage": 17.39, "elapsed_time": "0:00:10", "remaining_time": "0:00:47"}
+{"current_steps": 6, "total_steps": 23, "eval_loss": 0.6212631464004517, "epoch": 0.2608695652173913, "percentage": 26.09, "elapsed_time": "0:00:20", "remaining_time": "0:00:57"}
+{"current_steps": 8, "total_steps": 23, "eval_loss": 0.6109676957130432, "epoch": 0.34782608695652173, "percentage": 34.78, "elapsed_time": "0:00:27", "remaining_time": "0:00:50"}
+{"current_steps": 10, "total_steps": 23, "eval_loss": 0.5979903340339661, "epoch": 0.43478260869565216, "percentage": 43.48, "elapsed_time": "0:00:36", "remaining_time": "0:00:47"}
+{"current_steps": 12, "total_steps": 23, "eval_loss": 0.5886225700378418, "epoch": 0.5217391304347826, "percentage": 52.17, "elapsed_time": "0:00:40", "remaining_time": "0:00:37"}
+{"current_steps": 14, "total_steps": 23, "eval_loss": 0.5854005813598633, "epoch": 0.6086956521739131, "percentage": 60.87, "elapsed_time": "0:00:47", "remaining_time": "0:00:30"}
+{"current_steps": 16, "total_steps": 23, "eval_loss": 0.5828047394752502, "epoch": 0.6956521739130435, "percentage": 69.57, "elapsed_time": "0:00:53", "remaining_time": "0:00:23"}
+{"current_steps": 18, "total_steps": 23, "eval_loss": 0.5814995169639587, "epoch": 0.782608695652174, "percentage": 78.26, "elapsed_time": "0:00:59", "remaining_time": "0:00:16"}
+{"current_steps": 20, "total_steps": 23, "eval_loss": 0.5805516839027405, "epoch": 0.8695652173913043, "percentage": 86.96, "elapsed_time": "0:01:03", "remaining_time": "0:00:09"}
+{"current_steps": 22, "total_steps": 23, "eval_loss": 0.5803565382957458, "epoch": 0.9565217391304348, "percentage": 95.65, "elapsed_time": "0:01:10", "remaining_time": "0:00:03"}
+{"current_steps": 23, "total_steps": 23, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:01:26", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,131 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 2,
+  "global_step": 23,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.08695652173913043,
+      "eval_loss": 0.6561041474342346,
+      "eval_runtime": 1.4011,
+      "eval_samples_per_second": 3.569,
+      "eval_steps_per_second": 2.141,
+      "step": 2
+    },
+    {
+      "epoch": 0.17391304347826086,
+      "eval_loss": 0.6379454731941223,
+      "eval_runtime": 1.397,
+      "eval_samples_per_second": 3.579,
+      "eval_steps_per_second": 2.147,
+      "step": 4
+    },
+    {
+      "epoch": 0.2608695652173913,
+      "eval_loss": 0.6212631464004517,
+      "eval_runtime": 1.3983,
+      "eval_samples_per_second": 3.576,
+      "eval_steps_per_second": 2.146,
+      "step": 6
+    },
+    {
+      "epoch": 0.34782608695652173,
+      "eval_loss": 0.6109676957130432,
+      "eval_runtime": 1.396,
+      "eval_samples_per_second": 3.582,
+      "eval_steps_per_second": 2.149,
+      "step": 8
+    },
+    {
+      "epoch": 0.43478260869565216,
+      "eval_loss": 0.5979903340339661,
+      "eval_runtime": 1.3966,
+      "eval_samples_per_second": 3.58,
+      "eval_steps_per_second": 2.148,
+      "step": 10
+    },
+    {
+      "epoch": 0.5217391304347826,
+      "eval_loss": 0.5886225700378418,
+      "eval_runtime": 1.3975,
+      "eval_samples_per_second": 3.578,
+      "eval_steps_per_second": 2.147,
+      "step": 12
+    },
+    {
+      "epoch": 0.6086956521739131,
+      "eval_loss": 0.5854005813598633,
+      "eval_runtime": 1.3966,
+      "eval_samples_per_second": 3.58,
+      "eval_steps_per_second": 2.148,
+      "step": 14
+    },
+    {
+      "epoch": 0.6956521739130435,
+      "eval_loss": 0.5828047394752502,
+      "eval_runtime": 1.3966,
+      "eval_samples_per_second": 3.58,
+      "eval_steps_per_second": 2.148,
+      "step": 16
+    },
+    {
+      "epoch": 0.782608695652174,
+      "eval_loss": 0.5814995169639587,
+      "eval_runtime": 1.3995,
+      "eval_samples_per_second": 3.573,
+      "eval_steps_per_second": 2.144,
+      "step": 18
+    },
+    {
+      "epoch": 0.8695652173913043,
+      "eval_loss": 0.5805516839027405,
+      "eval_runtime": 1.3962,
+      "eval_samples_per_second": 3.581,
+      "eval_steps_per_second": 2.149,
+      "step": 20
+    },
+    {
+      "epoch": 0.9565217391304348,
+      "eval_loss": 0.5803565382957458,
+      "eval_runtime": 1.3957,
+      "eval_samples_per_second": 3.582,
+      "eval_steps_per_second": 2.149,
+      "step": 22
+    },
+    {
+      "epoch": 1.0,
+      "step": 23,
+      "total_flos": 331406114816.0,
+      "train_loss": 0.629167598226796,
+      "train_runtime": 86.9566,
+      "train_samples_per_second": 0.517,
+      "train_steps_per_second": 0.264
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 23,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 331406114816.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea7036ce228fe6229e5f3febb305ffd37ea2ad5626465abcc1f02f7d37460ac8
+size 7480