Upload folder using huggingface_hub

README.md

@@ -0,0 +1,161 @@
+---
+base_model: MangyMango/testing1
+tags:
+- generated_from_trainer
+model-index:
+- name: outputs/sd-prompter
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+base_model: MangyMango/testing1
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+
+trust_remote_code: true
+
+load_in_8bit: false
+load_in_4bit: false
+strict: false
+
+datasets:
+  - path: civit-slop-combined.jsonl
+    type: alpaca
+    conversation: mpt-30b-instruct
+
+chat_template: alpaca
+
+dataset_prepared_path:
+val_set_size: 0.05
+output_dir: ./outputs/sd-prompter
+sequence_len: 2048
+sample_packing: true
+eval_sample_packing: false
+pad_to_sequence_len: true
+
+adapter:
+lora_model_dir:
+lora_r:
+lora_alpha:
+lora_dropout:
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+wandb_project: Mango-SDprompt-qwen
+wandb_entity:
+wandb_watch:
+wandb_name: qwen1.5b-2
+wandb_log_model:
+
+gradient_accumulation_steps: 64
+micro_batch_size: 2
+num_epochs: 3
+optimizer: adamw_torch
+lr_scheduler: cosine
+learning_rate: 0.00002
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: true
+
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: false
+early_stopping_patience:
+resume_from_checkpoint:
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+warmup_ratio: 0.05
+evals_per_epoch: 4
+saves_per_epoch: 1
+debug:
+#deepspeed: deepspeed_configs/zero2.json
+#deepspeed: /training/axolotl/axolotl/deepspeed_configs/zero2.json
+weight_decay: 0.0
+#fsdp:
+#fsdp_config:
+#  fsdp_limit_all_gathers: true
+#  fsdp_sync_module_states: true
+#  fsdp_offload_params: true
+#  fsdp_use_orig_params: false
+#  fsdp_cpu_ram_efficient_loading: true
+#  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+#  fsdp_transformer_layer_cls_to_wrap: Qwen2DecoderLayer
+#  fsdp_state_dict_type: FULL_STATE_DICT
+special_tokens:
+
+```
+
+</details><br>
+
+# outputs/sd-prompter
+
+This model is a fine-tuned version of [MangyMango/testing1](https://huggingface.co/MangyMango/testing1) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 3.4889
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- gradient_accumulation_steps: 64
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 2
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 3.4783        | 0.0793 | 1    | 4.2312          |
+| 3.3803        | 0.2379 | 3    | 3.8651          |
+| 3.0646        | 0.4758 | 6    | 3.6872          |
+| 2.8913        | 0.7138 | 9    | 3.6106          |
+| 2.9159        | 0.9517 | 12   | 3.5590          |
+| 2.819         | 1.1660 | 15   | 3.5307          |
+| 2.8095        | 1.4040 | 18   | 3.5109          |
+| 2.8054        | 1.6419 | 21   | 3.4995          |
+| 2.9067        | 1.8798 | 24   | 3.4933          |
+| 2.8035        | 2.0954 | 27   | 3.4903          |
+| 2.7619        | 2.3333 | 30   | 3.4890          |
+| 2.8226        | 2.5713 | 33   | 3.4891          |
+| 2.7211        | 2.8092 | 36   | 3.4889          |
+
+
+### Framework versions
+
+- Transformers 4.44.0.dev0
+- Pytorch 2.1.2+cu118
+- Datasets 2.19.1
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

checkpoint-12/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

checkpoint-12/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "MangyMango/testing1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

checkpoint-12/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-12/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1c1ca701a4e6fdabadcbf513861989c0ac1188c42f1b68b7957ff3b70733189
+size 3087467144

checkpoint-12/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c062f7f375beded48b5337f5a3f3a5cb38807fa3e85dbf3e294c0ab6b627bfc2
+size 14244

checkpoint-12/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80e329610a4bb15c30b83e3a8ddf84c953fb5aa712605f525eb9d8d26527b9a8
+size 1064

checkpoint-12/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-12/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-12/trainer_state.json

@@ -0,0 +1,157 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9516728624535316,
+  "eval_steps": 3,
+  "global_step": 12,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07930607187112763,
+      "grad_norm": 16.125,
+      "learning_rate": 1e-05,
+      "loss": 3.4783,
+      "step": 1
+    },
+    {
+      "epoch": 0.07930607187112763,
+      "eval_loss": 4.231224060058594,
+      "eval_runtime": 35.8899,
+      "eval_samples_per_second": 15.436,
+      "eval_steps_per_second": 7.718,
+      "step": 1
+    },
+    {
+      "epoch": 0.15861214374225527,
+      "grad_norm": 15.875,
+      "learning_rate": 2e-05,
+      "loss": 3.581,
+      "step": 2
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.9957341762950346e-05,
+      "loss": 3.3803,
+      "step": 3
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "eval_loss": 3.8651065826416016,
+      "eval_runtime": 36.376,
+      "eval_samples_per_second": 15.23,
+      "eval_steps_per_second": 7.615,
+      "step": 3
+    },
+    {
+      "epoch": 0.31722428748451054,
+      "grad_norm": 9.0,
+      "learning_rate": 1.982973099683902e-05,
+      "loss": 3.2376,
+      "step": 4
+    },
+    {
+      "epoch": 0.3965303593556382,
+      "grad_norm": 6.34375,
+      "learning_rate": 1.961825643172819e-05,
+      "loss": 3.0861,
+      "step": 5
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.932472229404356e-05,
+      "loss": 3.0646,
+      "step": 6
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "eval_loss": 3.6871607303619385,
+      "eval_runtime": 36.6092,
+      "eval_samples_per_second": 15.133,
+      "eval_steps_per_second": 7.566,
+      "step": 6
+    },
+    {
+      "epoch": 0.5551425030978935,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.8951632913550625e-05,
+      "loss": 3.0159,
+      "step": 7
+    },
+    {
+      "epoch": 0.6344485749690211,
+      "grad_norm": 2.75,
+      "learning_rate": 1.8502171357296144e-05,
+      "loss": 3.0312,
+      "step": 8
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.7980172272802398e-05,
+      "loss": 2.8913,
+      "step": 9
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "eval_loss": 3.6105825901031494,
+      "eval_runtime": 36.8528,
+      "eval_samples_per_second": 15.033,
+      "eval_steps_per_second": 7.516,
+      "step": 9
+    },
+    {
+      "epoch": 0.7930607187112764,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.7390089172206594e-05,
+      "loss": 3.0189,
+      "step": 10
+    },
+    {
+      "epoch": 0.872366790582404,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.6736956436465573e-05,
+      "loss": 2.8854,
+      "step": 11
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.6026346363792565e-05,
+      "loss": 2.9159,
+      "step": 12
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "eval_loss": 3.558952808380127,
+      "eval_runtime": 36.3425,
+      "eval_samples_per_second": 15.244,
+      "eval_steps_per_second": 7.622,
+      "step": 12
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 36,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.4731850840735744e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-12/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62727f4a7a4a177da52e1c1a5a3c8e77bd73a77d298b43afdc5dae9d8285214e
+size 6136

checkpoint-24/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

checkpoint-24/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "MangyMango/testing1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

checkpoint-24/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-24/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90007ac5ce85e3b9a42fb61cd5d9eb9486061590012120a01f7889bd021bfb5a
+size 3087467144

checkpoint-24/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5edb34d031c0c2b447f3eaadb401a4c1e7e7e6d8c096e28b7092e01a8bd48c92
+size 14244

checkpoint-24/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3189f1ed324bd36e014e7dc9cf646f7f1fb36cffaed81bcec474b9b6602ea139
+size 1064

checkpoint-24/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-24/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-24/trainer_state.json

@@ -0,0 +1,273 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.8798017348203222,
+  "eval_steps": 3,
+  "global_step": 24,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07930607187112763,
+      "grad_norm": 16.125,
+      "learning_rate": 1e-05,
+      "loss": 3.4783,
+      "step": 1
+    },
+    {
+      "epoch": 0.07930607187112763,
+      "eval_loss": 4.231224060058594,
+      "eval_runtime": 35.8899,
+      "eval_samples_per_second": 15.436,
+      "eval_steps_per_second": 7.718,
+      "step": 1
+    },
+    {
+      "epoch": 0.15861214374225527,
+      "grad_norm": 15.875,
+      "learning_rate": 2e-05,
+      "loss": 3.581,
+      "step": 2
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.9957341762950346e-05,
+      "loss": 3.3803,
+      "step": 3
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "eval_loss": 3.8651065826416016,
+      "eval_runtime": 36.376,
+      "eval_samples_per_second": 15.23,
+      "eval_steps_per_second": 7.615,
+      "step": 3
+    },
+    {
+      "epoch": 0.31722428748451054,
+      "grad_norm": 9.0,
+      "learning_rate": 1.982973099683902e-05,
+      "loss": 3.2376,
+      "step": 4
+    },
+    {
+      "epoch": 0.3965303593556382,
+      "grad_norm": 6.34375,
+      "learning_rate": 1.961825643172819e-05,
+      "loss": 3.0861,
+      "step": 5
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.932472229404356e-05,
+      "loss": 3.0646,
+      "step": 6
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "eval_loss": 3.6871607303619385,
+      "eval_runtime": 36.6092,
+      "eval_samples_per_second": 15.133,
+      "eval_steps_per_second": 7.566,
+      "step": 6
+    },
+    {
+      "epoch": 0.5551425030978935,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.8951632913550625e-05,
+      "loss": 3.0159,
+      "step": 7
+    },
+    {
+      "epoch": 0.6344485749690211,
+      "grad_norm": 2.75,
+      "learning_rate": 1.8502171357296144e-05,
+      "loss": 3.0312,
+      "step": 8
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.7980172272802398e-05,
+      "loss": 2.8913,
+      "step": 9
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "eval_loss": 3.6105825901031494,
+      "eval_runtime": 36.8528,
+      "eval_samples_per_second": 15.033,
+      "eval_steps_per_second": 7.516,
+      "step": 9
+    },
+    {
+      "epoch": 0.7930607187112764,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.7390089172206594e-05,
+      "loss": 3.0189,
+      "step": 10
+    },
+    {
+      "epoch": 0.872366790582404,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.6736956436465573e-05,
+      "loss": 2.8854,
+      "step": 11
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.6026346363792565e-05,
+      "loss": 2.9159,
+      "step": 12
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "eval_loss": 3.558952808380127,
+      "eval_runtime": 36.3425,
+      "eval_samples_per_second": 15.244,
+      "eval_steps_per_second": 7.622,
+      "step": 12
+    },
+    {
+      "epoch": 1.0074349442379182,
+      "grad_norm": 1.625,
+      "learning_rate": 1.526432162877356e-05,
+      "loss": 2.9194,
+      "step": 13
+    },
+    {
+      "epoch": 1.0867410161090458,
+      "grad_norm": 1.5,
+      "learning_rate": 1.4457383557765385e-05,
+      "loss": 2.8288,
+      "step": 14
+    },
+    {
+      "epoch": 1.1660470879801734,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.3612416661871532e-05,
+      "loss": 2.819,
+      "step": 15
+    },
+    {
+      "epoch": 1.1660470879801734,
+      "eval_loss": 3.530693531036377,
+      "eval_runtime": 36.8292,
+      "eval_samples_per_second": 15.042,
+      "eval_steps_per_second": 7.521,
+      "step": 15
+    },
+    {
+      "epoch": 1.2453531598513012,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.2736629900720832e-05,
+      "loss": 2.9035,
+      "step": 16
+    },
+    {
+      "epoch": 1.3246592317224288,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.1837495178165706e-05,
+      "loss": 2.8615,
+      "step": 17
+    },
+    {
+      "epoch": 1.4039653035935564,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.092268359463302e-05,
+      "loss": 2.8095,
+      "step": 18
+    },
+    {
+      "epoch": 1.4039653035935564,
+      "eval_loss": 3.5109074115753174,
+      "eval_runtime": 36.5179,
+      "eval_samples_per_second": 15.171,
+      "eval_steps_per_second": 7.585,
+      "step": 18
+    },
+    {
+      "epoch": 1.483271375464684,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1e-05,
+      "loss": 2.7651,
+      "step": 19
+    },
+    {
+      "epoch": 1.5625774473358116,
+      "grad_norm": 1.3828125,
+      "learning_rate": 9.07731640536698e-06,
+      "loss": 2.8122,
+      "step": 20
+    },
+    {
+      "epoch": 1.6418835192069392,
+      "grad_norm": 1.390625,
+      "learning_rate": 8.162504821834296e-06,
+      "loss": 2.8054,
+      "step": 21
+    },
+    {
+      "epoch": 1.6418835192069392,
+      "eval_loss": 3.499537706375122,
+      "eval_runtime": 36.564,
+      "eval_samples_per_second": 15.152,
+      "eval_steps_per_second": 7.576,
+      "step": 21
+    },
+    {
+      "epoch": 1.7211895910780668,
+      "grad_norm": 1.296875,
+      "learning_rate": 7.263370099279173e-06,
+      "loss": 2.7538,
+      "step": 22
+    },
+    {
+      "epoch": 1.8004956629491944,
+      "grad_norm": 1.328125,
+      "learning_rate": 6.387583338128471e-06,
+      "loss": 2.7587,
+      "step": 23
+    },
+    {
+      "epoch": 1.8798017348203222,
+      "grad_norm": 1.3359375,
+      "learning_rate": 5.542616442234618e-06,
+      "loss": 2.9067,
+      "step": 24
+    },
+    {
+      "epoch": 1.8798017348203222,
+      "eval_loss": 3.4932873249053955,
+      "eval_runtime": 36.4531,
+      "eval_samples_per_second": 15.198,
+      "eval_steps_per_second": 7.599,
+      "step": 24
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 36,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.944760021608038e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-24/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62727f4a7a4a177da52e1c1a5a3c8e77bd73a77d298b43afdc5dae9d8285214e
+size 6136

checkpoint-36/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

checkpoint-36/config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "MangyMango/testing1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

checkpoint-36/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

checkpoint-36/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98fffd2fe1c11b395d50bf844bafaae8a1d0973dbedf46f1b1ebadeca639c50
+size 3087467144

checkpoint-36/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52cca5856c568bc52c683b690919168fa27bfbdfefc6e0a62355afa6011157c3
+size 14244

checkpoint-36/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fd2743287d156cd04c7839d1c540f41c34a34bc9c901ef1a26c3825917fc7aa
+size 1064

checkpoint-36/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-36/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

checkpoint-36/trainer_state.json

@@ -0,0 +1,389 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.809169764560099,
+  "eval_steps": 3,
+  "global_step": 36,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.07930607187112763,
+      "grad_norm": 16.125,
+      "learning_rate": 1e-05,
+      "loss": 3.4783,
+      "step": 1
+    },
+    {
+      "epoch": 0.07930607187112763,
+      "eval_loss": 4.231224060058594,
+      "eval_runtime": 35.8899,
+      "eval_samples_per_second": 15.436,
+      "eval_steps_per_second": 7.718,
+      "step": 1
+    },
+    {
+      "epoch": 0.15861214374225527,
+      "grad_norm": 15.875,
+      "learning_rate": 2e-05,
+      "loss": 3.581,
+      "step": 2
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "grad_norm": 8.5625,
+      "learning_rate": 1.9957341762950346e-05,
+      "loss": 3.3803,
+      "step": 3
+    },
+    {
+      "epoch": 0.2379182156133829,
+      "eval_loss": 3.8651065826416016,
+      "eval_runtime": 36.376,
+      "eval_samples_per_second": 15.23,
+      "eval_steps_per_second": 7.615,
+      "step": 3
+    },
+    {
+      "epoch": 0.31722428748451054,
+      "grad_norm": 9.0,
+      "learning_rate": 1.982973099683902e-05,
+      "loss": 3.2376,
+      "step": 4
+    },
+    {
+      "epoch": 0.3965303593556382,
+      "grad_norm": 6.34375,
+      "learning_rate": 1.961825643172819e-05,
+      "loss": 3.0861,
+      "step": 5
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "grad_norm": 3.5625,
+      "learning_rate": 1.932472229404356e-05,
+      "loss": 3.0646,
+      "step": 6
+    },
+    {
+      "epoch": 0.4758364312267658,
+      "eval_loss": 3.6871607303619385,
+      "eval_runtime": 36.6092,
+      "eval_samples_per_second": 15.133,
+      "eval_steps_per_second": 7.566,
+      "step": 6
+    },
+    {
+      "epoch": 0.5551425030978935,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.8951632913550625e-05,
+      "loss": 3.0159,
+      "step": 7
+    },
+    {
+      "epoch": 0.6344485749690211,
+      "grad_norm": 2.75,
+      "learning_rate": 1.8502171357296144e-05,
+      "loss": 3.0312,
+      "step": 8
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.7980172272802398e-05,
+      "loss": 2.8913,
+      "step": 9
+    },
+    {
+      "epoch": 0.7137546468401487,
+      "eval_loss": 3.6105825901031494,
+      "eval_runtime": 36.8528,
+      "eval_samples_per_second": 15.033,
+      "eval_steps_per_second": 7.516,
+      "step": 9
+    },
+    {
+      "epoch": 0.7930607187112764,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.7390089172206594e-05,
+      "loss": 3.0189,
+      "step": 10
+    },
+    {
+      "epoch": 0.872366790582404,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.6736956436465573e-05,
+      "loss": 2.8854,
+      "step": 11
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.6026346363792565e-05,
+      "loss": 2.9159,
+      "step": 12
+    },
+    {
+      "epoch": 0.9516728624535316,
+      "eval_loss": 3.558952808380127,
+      "eval_runtime": 36.3425,
+      "eval_samples_per_second": 15.244,
+      "eval_steps_per_second": 7.622,
+      "step": 12
+    },
+    {
+      "epoch": 1.0074349442379182,
+      "grad_norm": 1.625,
+      "learning_rate": 1.526432162877356e-05,
+      "loss": 2.9194,
+      "step": 13
+    },
+    {
+      "epoch": 1.0867410161090458,
+      "grad_norm": 1.5,
+      "learning_rate": 1.4457383557765385e-05,
+      "loss": 2.8288,
+      "step": 14
+    },
+    {
+      "epoch": 1.1660470879801734,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.3612416661871532e-05,
+      "loss": 2.819,
+      "step": 15
+    },
+    {
+      "epoch": 1.1660470879801734,
+      "eval_loss": 3.530693531036377,
+      "eval_runtime": 36.8292,
+      "eval_samples_per_second": 15.042,
+      "eval_steps_per_second": 7.521,
+      "step": 15
+    },
+    {
+      "epoch": 1.2453531598513012,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.2736629900720832e-05,
+      "loss": 2.9035,
+      "step": 16
+    },
+    {
+      "epoch": 1.3246592317224288,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.1837495178165706e-05,
+      "loss": 2.8615,
+      "step": 17
+    },
+    {
+      "epoch": 1.4039653035935564,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.092268359463302e-05,
+      "loss": 2.8095,
+      "step": 18
+    },
+    {
+      "epoch": 1.4039653035935564,
+      "eval_loss": 3.5109074115753174,
+      "eval_runtime": 36.5179,
+      "eval_samples_per_second": 15.171,
+      "eval_steps_per_second": 7.585,
+      "step": 18
+    },
+    {
+      "epoch": 1.483271375464684,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1e-05,
+      "loss": 2.7651,
+      "step": 19
+    },
+    {
+      "epoch": 1.5625774473358116,
+      "grad_norm": 1.3828125,
+      "learning_rate": 9.07731640536698e-06,
+      "loss": 2.8122,
+      "step": 20
+    },
+    {
+      "epoch": 1.6418835192069392,
+      "grad_norm": 1.390625,
+      "learning_rate": 8.162504821834296e-06,
+      "loss": 2.8054,
+      "step": 21
+    },
+    {
+      "epoch": 1.6418835192069392,
+      "eval_loss": 3.499537706375122,
+      "eval_runtime": 36.564,
+      "eval_samples_per_second": 15.152,
+      "eval_steps_per_second": 7.576,
+      "step": 21
+    },
+    {
+      "epoch": 1.7211895910780668,
+      "grad_norm": 1.296875,
+      "learning_rate": 7.263370099279173e-06,
+      "loss": 2.7538,
+      "step": 22
+    },
+    {
+      "epoch": 1.8004956629491944,
+      "grad_norm": 1.328125,
+      "learning_rate": 6.387583338128471e-06,
+      "loss": 2.7587,
+      "step": 23
+    },
+    {
+      "epoch": 1.8798017348203222,
+      "grad_norm": 1.3359375,
+      "learning_rate": 5.542616442234618e-06,
+      "loss": 2.9067,
+      "step": 24
+    },
+    {
+      "epoch": 1.8798017348203222,
+      "eval_loss": 3.4932873249053955,
+      "eval_runtime": 36.4531,
+      "eval_samples_per_second": 15.198,
+      "eval_steps_per_second": 7.599,
+      "step": 24
+    },
+    {
+      "epoch": 1.9591078066914498,
+      "grad_norm": 1.296875,
+      "learning_rate": 4.7356783712264405e-06,
+      "loss": 2.7533,
+      "step": 25
+    },
+    {
+      "epoch": 2.0161090458488227,
+      "grad_norm": 1.3125,
+      "learning_rate": 3.973653636207437e-06,
+      "loss": 2.7047,
+      "step": 26
+    },
+    {
+      "epoch": 2.0954151177199503,
+      "grad_norm": 1.25,
+      "learning_rate": 3.2630435635344283e-06,
+      "loss": 2.8035,
+      "step": 27
+    },
+    {
+      "epoch": 2.0954151177199503,
+      "eval_loss": 3.4902639389038086,
+      "eval_runtime": 36.5582,
+      "eval_samples_per_second": 15.154,
+      "eval_steps_per_second": 7.577,
+      "step": 27
+    },
+    {
+      "epoch": 2.174721189591078,
+      "grad_norm": 1.2890625,
+      "learning_rate": 2.6099108277934105e-06,
+      "loss": 2.778,
+      "step": 28
+    },
+    {
+      "epoch": 2.254027261462206,
+      "grad_norm": 1.2421875,
+      "learning_rate": 2.019827727197605e-06,
+      "loss": 2.691,
+      "step": 29
+    },
+    {
+      "epoch": 2.3333333333333335,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.4978286427038602e-06,
+      "loss": 2.7619,
+      "step": 30
+    },
+    {
+      "epoch": 2.3333333333333335,
+      "eval_loss": 3.4889628887176514,
+      "eval_runtime": 36.371,
+      "eval_samples_per_second": 15.232,
+      "eval_steps_per_second": 7.616,
+      "step": 30
+    },
+    {
+      "epoch": 2.412639405204461,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.0483670864493777e-06,
+      "loss": 2.7703,
+      "step": 31
+    },
+    {
+      "epoch": 2.4919454770755887,
+      "grad_norm": 1.2734375,
+      "learning_rate": 6.752777059564431e-07,
+      "loss": 2.8694,
+      "step": 32
+    },
+    {
+      "epoch": 2.5712515489467163,
+      "grad_norm": 1.359375,
+      "learning_rate": 3.817435682718096e-07,
+      "loss": 2.8226,
+      "step": 33
+    },
+    {
+      "epoch": 2.5712515489467163,
+      "eval_loss": 3.489114999771118,
+      "eval_runtime": 36.366,
+      "eval_samples_per_second": 15.234,
+      "eval_steps_per_second": 7.617,
+      "step": 33
+    },
+    {
+      "epoch": 2.650557620817844,
+      "grad_norm": 1.296875,
+      "learning_rate": 1.7026900316098217e-07,
+      "loss": 2.7609,
+      "step": 34
+    },
+    {
+      "epoch": 2.7298636926889714,
+      "grad_norm": 1.28125,
+      "learning_rate": 4.2658237049655325e-08,
+      "loss": 2.7784,
+      "step": 35
+    },
+    {
+      "epoch": 2.809169764560099,
+      "grad_norm": 1.296875,
+      "learning_rate": 0.0,
+      "loss": 2.7211,
+      "step": 36
+    },
+    {
+      "epoch": 2.809169764560099,
+      "eval_loss": 3.488926649093628,
+      "eval_runtime": 36.5143,
+      "eval_samples_per_second": 15.172,
+      "eval_steps_per_second": 7.586,
+      "step": 36
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 36,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 12,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.417945105681613e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-36/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62727f4a7a4a177da52e1c1a5a3c8e77bd73a77d298b43afdc5dae9d8285214e
+size 6136

config.json

@@ -0,0 +1,27 @@
+{
+  "_name_or_path": "MangyMango/testing1",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 28,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.44.0.dev0"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccf55c7006fe947a822f93af5fbf9b11c7024c4e4c4468efdcf681d9d82b14bc
+size 3087542418

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}