Upload 16 files

README.md

@@ -0,0 +1,58 @@
+---
+base_model: mistralai/Mistral-7B-Instruct-v0.1
+library_name: peft
+license: other
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: mistral_docker
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# mistral_docker
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.1](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1) on the minecraft dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.43.4
+- Pytorch 2.2.1
+- Datasets 2.21.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.1",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "up_proj",
+    "k_proj",
+    "q_proj",
+    "o_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afb161c26ed8b4feebed547a41efca0e1e9a87608e30b343d6e6312fa5c662c6
+size 83945296

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "num_input_tokens_seen": 437216,
+    "total_flos": 1.8708380691726336e+16,
+    "train_loss": 0.044493223137528556,
+    "train_runtime": 988.3228,
+    "train_samples_per_second": 7.331,
+    "train_steps_per_second": 0.458
+}

llamaboard_config.yaml

@@ -0,0 +1,67 @@
+top.booster: auto
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: Mistral-7B-v0.1-Chat
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: mistral
+top.visual_inputs: false
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 2
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 1024
+train.dataset:
+- minecraft
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-5
+train.logging_steps: 5
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '1.0'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '3.0'
+train.optim: adamw_torch
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 100
+train.shift_attn: false
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0
+train.warmup_steps: 0

running_log.txt

@@ -0,0 +1,510 @@
+[INFO|parser.py:355] 2024-08-29 19:20:42,528 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, compute dtype: torch.bfloat16
+
+[INFO|tokenization_utils_base.py:2289] 2024-08-29 19:20:43,824 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/tokenizer.model
+
+[INFO|tokenization_utils_base.py:2289] 2024-08-29 19:20:43,824 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/tokenizer.json
+
+[INFO|tokenization_utils_base.py:2289] 2024-08-29 19:20:43,824 >> loading file added_tokens.json from cache at None
+
+[INFO|tokenization_utils_base.py:2289] 2024-08-29 19:20:43,824 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/special_tokens_map.json
+
+[INFO|tokenization_utils_base.py:2289] 2024-08-29 19:20:43,825 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/tokenizer_config.json
+
+[INFO|template.py:373] 2024-08-29 19:20:43,869 >> Add pad token: </s>
+
+[INFO|loader.py:52] 2024-08-29 19:20:43,870 >> Loading dataset MattCoddity/dockerNLcommands...
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:20:46,934 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:20:46,935 >> Model config MistralConfig {
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|modeling_utils.py:3644] 2024-08-29 19:20:47,387 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/model.safetensors.index.json
+
+[INFO|modeling_utils.py:1572] 2024-08-29 19:22:01,945 >> Instantiating MistralForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|configuration_utils.py:1038] 2024-08-29 19:22:01,946 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+[INFO|modeling_utils.py:4473] 2024-08-29 19:22:05,773 >> All model checkpoint weights were used when initializing MistralForCausalLM.
+
+
+[INFO|modeling_utils.py:4481] 2024-08-29 19:22:05,773 >> All the weights of MistralForCausalLM were initialized from the model checkpoint at mistralai/Mistral-7B-Instruct-v0.1.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralForCausalLM for predictions without further training.
+
+[INFO|configuration_utils.py:993] 2024-08-29 19:22:06,005 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/generation_config.json
+
+[INFO|configuration_utils.py:1038] 2024-08-29 19:22:06,005 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+[INFO|checkpointing.py:103] 2024-08-29 19:22:06,012 >> Gradient checkpointing enabled.
+
+[INFO|attention.py:84] 2024-08-29 19:22:06,012 >> Using torch SDPA for faster training and inference.
+
+[INFO|adapter.py:302] 2024-08-29 19:22:06,012 >> Upcasting trainable params to float32.
+
+[INFO|adapter.py:158] 2024-08-29 19:22:06,012 >> Fine-tuning method: LoRA
+
+[INFO|misc.py:51] 2024-08-29 19:22:06,013 >> Found linear modules: gate_proj,v_proj,up_proj,k_proj,q_proj,o_proj,down_proj
+
+[INFO|loader.py:196] 2024-08-29 19:22:21,642 >> trainable params: 20,971,520 || all params: 7,262,703,616 || trainable%: 0.2888
+
+[INFO|trainer.py:648] 2024-08-29 19:22:21,652 >> Using auto half precision backend
+
+[INFO|trainer.py:2134] 2024-08-29 19:22:22,077 >> ***** Running training *****
+
+[INFO|trainer.py:2135] 2024-08-29 19:22:22,077 >>   Num examples = 2,415
+
+[INFO|trainer.py:2136] 2024-08-29 19:22:22,077 >>   Num Epochs = 3
+
+[INFO|trainer.py:2137] 2024-08-29 19:22:22,077 >>   Instantaneous batch size per device = 2
+
+[INFO|trainer.py:2140] 2024-08-29 19:22:22,077 >>   Total train batch size (w. parallel, distributed & accumulation) = 16
+
+[INFO|trainer.py:2141] 2024-08-29 19:22:22,078 >>   Gradient Accumulation steps = 8
+
+[INFO|trainer.py:2142] 2024-08-29 19:22:22,078 >>   Total optimization steps = 453
+
+[INFO|trainer.py:2143] 2024-08-29 19:22:22,081 >>   Number of trainable parameters = 20,971,520
+
+[INFO|callbacks.py:319] 2024-08-29 19:22:33,541 >> {'loss': 0.6997, 'learning_rate': 4.9985e-05, 'epoch': 0.03, 'throughput': 428.77}
+
+[INFO|callbacks.py:319] 2024-08-29 19:22:44,569 >> {'loss': 0.3509, 'learning_rate': 4.9940e-05, 'epoch': 0.07, 'throughput': 441.91}
+
+[INFO|callbacks.py:319] 2024-08-29 19:22:55,574 >> {'loss': 0.2853, 'learning_rate': 4.9865e-05, 'epoch': 0.10, 'throughput': 443.85}
+
+[INFO|callbacks.py:319] 2024-08-29 19:23:06,256 >> {'loss': 0.2456, 'learning_rate': 4.9760e-05, 'epoch': 0.13, 'throughput': 448.8}
+
+[INFO|callbacks.py:319] 2024-08-29 19:23:17,193 >> {'loss': 0.2084, 'learning_rate': 4.9625e-05, 'epoch': 0.17, 'throughput': 444.22}
+
+[INFO|callbacks.py:319] 2024-08-29 19:23:28,050 >> {'loss': 0.1626, 'learning_rate': 4.9461e-05, 'epoch': 0.20, 'throughput': 444.12}
+
+[INFO|callbacks.py:319] 2024-08-29 19:23:38,739 >> {'loss': 0.1405, 'learning_rate': 4.9267e-05, 'epoch': 0.23, 'throughput': 442.72}
+
+[INFO|callbacks.py:319] 2024-08-29 19:23:49,936 >> {'loss': 0.1182, 'learning_rate': 4.9044e-05, 'epoch': 0.26, 'throughput': 444.57}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:01,393 >> {'loss': 0.1285, 'learning_rate': 4.8792e-05, 'epoch': 0.30, 'throughput': 438.23}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:12,547 >> {'loss': 0.0906, 'learning_rate': 4.8512e-05, 'epoch': 0.33, 'throughput': 440.48}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:23,416 >> {'loss': 0.0540, 'learning_rate': 4.8203e-05, 'epoch': 0.36, 'throughput': 441.77}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:34,155 >> {'loss': 0.0730, 'learning_rate': 4.7867e-05, 'epoch': 0.40, 'throughput': 442.07}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:44,702 >> {'loss': 0.0524, 'learning_rate': 4.7503e-05, 'epoch': 0.43, 'throughput': 442.81}
+
+[INFO|callbacks.py:319] 2024-08-29 19:24:55,348 >> {'loss': 0.0534, 'learning_rate': 4.7112e-05, 'epoch': 0.46, 'throughput': 442.95}
+
+[INFO|callbacks.py:319] 2024-08-29 19:25:06,024 >> {'loss': 0.0759, 'learning_rate': 4.6694e-05, 'epoch': 0.50, 'throughput': 442.11}
+
+[INFO|callbacks.py:319] 2024-08-29 19:25:16,998 >> {'loss': 0.0766, 'learning_rate': 4.6250e-05, 'epoch': 0.53, 'throughput': 440.9}
+
+[INFO|callbacks.py:319] 2024-08-29 19:25:28,034 >> {'loss': 0.0383, 'learning_rate': 4.5781e-05, 'epoch': 0.56, 'throughput': 438.74}
+
+[INFO|callbacks.py:319] 2024-08-29 19:25:38,775 >> {'loss': 0.0393, 'learning_rate': 4.5286e-05, 'epoch': 0.60, 'throughput': 439.35}
+
+[INFO|callbacks.py:319] 2024-08-29 19:25:49,280 >> {'loss': 0.0589, 'learning_rate': 4.4768e-05, 'epoch': 0.63, 'throughput': 440.4}
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:00,073 >> {'loss': 0.0351, 'learning_rate': 4.4225e-05, 'epoch': 0.66, 'throughput': 440.1}
+
+[INFO|trainer.py:3503] 2024-08-29 19:26:00,082 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-100
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:26:00,312 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:26:00,313 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:26:00,409 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-100/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:26:00,409 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-100/special_tokens_map.json
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:11,158 >> {'loss': 0.0417, 'learning_rate': 4.3660e-05, 'epoch': 0.70, 'throughput': 439.61}
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:22,031 >> {'loss': 0.0437, 'learning_rate': 4.3072e-05, 'epoch': 0.73, 'throughput': 439.17}
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:33,043 >> {'loss': 0.0498, 'learning_rate': 4.2462e-05, 'epoch': 0.76, 'throughput': 439.15}
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:43,782 >> {'loss': 0.0239, 'learning_rate': 4.1831e-05, 'epoch': 0.79, 'throughput': 438.31}
+
+[INFO|callbacks.py:319] 2024-08-29 19:26:54,491 >> {'loss': 0.0228, 'learning_rate': 4.1180e-05, 'epoch': 0.83, 'throughput': 439.58}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:05,490 >> {'loss': 0.0390, 'learning_rate': 4.0510e-05, 'epoch': 0.86, 'throughput': 440.13}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:16,280 >> {'loss': 0.0500, 'learning_rate': 3.9821e-05, 'epoch': 0.89, 'throughput': 440.36}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:26,762 >> {'loss': 0.0200, 'learning_rate': 3.9114e-05, 'epoch': 0.93, 'throughput': 441.12}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:37,562 >> {'loss': 0.0159, 'learning_rate': 3.8390e-05, 'epoch': 0.96, 'throughput': 441.95}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:48,075 >> {'loss': 0.0170, 'learning_rate': 3.7650e-05, 'epoch': 0.99, 'throughput': 442.86}
+
+[INFO|callbacks.py:319] 2024-08-29 19:27:58,470 >> {'loss': 0.0298, 'learning_rate': 3.6895e-05, 'epoch': 1.03, 'throughput': 443.06}
+
+[INFO|callbacks.py:319] 2024-08-29 19:28:09,021 >> {'loss': 0.0357, 'learning_rate': 3.6125e-05, 'epoch': 1.06, 'throughput': 443.05}
+
+[INFO|callbacks.py:319] 2024-08-29 19:28:19,371 >> {'loss': 0.0185, 'learning_rate': 3.5342e-05, 'epoch': 1.09, 'throughput': 444.01}
+
+[INFO|callbacks.py:319] 2024-08-29 19:28:30,281 >> {'loss': 0.0339, 'learning_rate': 3.4547e-05, 'epoch': 1.13, 'throughput': 444.2}
+
+[INFO|callbacks.py:319] 2024-08-29 19:28:41,235 >> {'loss': 0.0224, 'learning_rate': 3.3740e-05, 'epoch': 1.16, 'throughput': 444.02}
+
+[INFO|callbacks.py:319] 2024-08-29 19:28:51,823 >> {'loss': 0.0192, 'learning_rate': 3.2923e-05, 'epoch': 1.19, 'throughput': 444.69}
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:02,724 >> {'loss': 0.0275, 'learning_rate': 3.2096e-05, 'epoch': 1.23, 'throughput': 444.69}
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:13,570 >> {'loss': 0.0086, 'learning_rate': 3.1261e-05, 'epoch': 1.26, 'throughput': 444.59}
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:24,314 >> {'loss': 0.0166, 'learning_rate': 3.0418e-05, 'epoch': 1.29, 'throughput': 445.07}
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:35,290 >> {'loss': 0.0167, 'learning_rate': 2.9569e-05, 'epoch': 1.32, 'throughput': 445.5}
+
+[INFO|trainer.py:3503] 2024-08-29 19:29:35,291 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-200
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:29:35,516 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:29:35,516 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:29:35,602 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-200/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:29:35,602 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-200/special_tokens_map.json
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:46,672 >> {'loss': 0.0137, 'learning_rate': 2.8714e-05, 'epoch': 1.36, 'throughput': 444.49}
+
+[INFO|callbacks.py:319] 2024-08-29 19:29:57,486 >> {'loss': 0.0142, 'learning_rate': 2.7854e-05, 'epoch': 1.39, 'throughput': 444.44}
+
+[INFO|callbacks.py:319] 2024-08-29 19:30:08,468 >> {'loss': 0.0136, 'learning_rate': 2.6992e-05, 'epoch': 1.42, 'throughput': 444.3}
+
+[INFO|callbacks.py:319] 2024-08-29 19:30:19,208 >> {'loss': 0.0184, 'learning_rate': 2.6127e-05, 'epoch': 1.46, 'throughput': 443.79}
+
+[INFO|callbacks.py:319] 2024-08-29 19:30:30,135 >> {'loss': 0.0098, 'learning_rate': 2.5260e-05, 'epoch': 1.49, 'throughput': 443.82}
+
+[INFO|callbacks.py:319] 2024-08-29 19:30:40,756 >> {'loss': 0.0120, 'learning_rate': 2.4393e-05, 'epoch': 1.52, 'throughput': 444.28}
+
+[INFO|callbacks.py:319] 2024-08-29 19:30:51,882 >> {'loss': 0.0106, 'learning_rate': 2.3527e-05, 'epoch': 1.56, 'throughput': 443.85}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:02,845 >> {'loss': 0.0128, 'learning_rate': 2.2663e-05, 'epoch': 1.59, 'throughput': 443.72}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:13,810 >> {'loss': 0.0139, 'learning_rate': 2.1801e-05, 'epoch': 1.62, 'throughput': 443.33}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:24,525 >> {'loss': 0.0084, 'learning_rate': 2.0944e-05, 'epoch': 1.66, 'throughput': 443.48}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:35,394 >> {'loss': 0.0143, 'learning_rate': 2.0091e-05, 'epoch': 1.69, 'throughput': 443.53}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:46,123 >> {'loss': 0.0110, 'learning_rate': 1.9244e-05, 'epoch': 1.72, 'throughput': 443.94}
+
+[INFO|callbacks.py:319] 2024-08-29 19:31:57,253 >> {'loss': 0.0127, 'learning_rate': 1.8404e-05, 'epoch': 1.75, 'throughput': 443.72}
+
+[INFO|callbacks.py:319] 2024-08-29 19:32:07,949 >> {'loss': 0.0127, 'learning_rate': 1.7572e-05, 'epoch': 1.79, 'throughput': 443.93}
+
+[INFO|callbacks.py:319] 2024-08-29 19:32:18,963 >> {'loss': 0.0122, 'learning_rate': 1.6749e-05, 'epoch': 1.82, 'throughput': 443.72}
+
+[INFO|callbacks.py:319] 2024-08-29 19:32:29,997 >> {'loss': 0.0169, 'learning_rate': 1.5936e-05, 'epoch': 1.85, 'throughput': 443.77}
+
+[INFO|callbacks.py:319] 2024-08-29 19:32:40,825 >> {'loss': 0.0242, 'learning_rate': 1.5133e-05, 'epoch': 1.89, 'throughput': 444.05}
+
+[INFO|callbacks.py:319] 2024-08-29 19:32:51,740 >> {'loss': 0.0079, 'learning_rate': 1.4343e-05, 'epoch': 1.92, 'throughput': 443.8}
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:03,290 >> {'loss': 0.0194, 'learning_rate': 1.3565e-05, 'epoch': 1.95, 'throughput': 443.36}
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:14,813 >> {'loss': 0.0133, 'learning_rate': 1.2801e-05, 'epoch': 1.99, 'throughput': 443.09}
+
+[INFO|trainer.py:3503] 2024-08-29 19:33:14,814 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-300
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:33:15,036 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:33:15,037 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:33:15,117 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-300/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:33:15,117 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-300/special_tokens_map.json
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:26,589 >> {'loss': 0.0049, 'learning_rate': 1.2052e-05, 'epoch': 2.02, 'throughput': 442.42}
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:37,297 >> {'loss': 0.0121, 'learning_rate': 1.1319e-05, 'epoch': 2.05, 'throughput': 442.9}
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:47,964 >> {'loss': 0.0102, 'learning_rate': 1.0601e-05, 'epoch': 2.09, 'throughput': 442.89}
+
+[INFO|callbacks.py:319] 2024-08-29 19:33:59,114 >> {'loss': 0.0072, 'learning_rate': 9.9016e-06, 'epoch': 2.12, 'throughput': 443.13}
+
+[INFO|callbacks.py:319] 2024-08-29 19:34:10,116 >> {'loss': 0.0066, 'learning_rate': 9.2199e-06, 'epoch': 2.15, 'throughput': 443.2}
+
+[INFO|callbacks.py:319] 2024-08-29 19:34:21,353 >> {'loss': 0.0119, 'learning_rate': 8.5571e-06, 'epoch': 2.19, 'throughput': 442.86}
+
+[INFO|callbacks.py:319] 2024-08-29 19:34:32,508 >> {'loss': 0.0057, 'learning_rate': 7.9141e-06, 'epoch': 2.22, 'throughput': 442.91}
+
+[INFO|callbacks.py:319] 2024-08-29 19:34:43,266 >> {'loss': 0.0136, 'learning_rate': 7.2917e-06, 'epoch': 2.25, 'throughput': 443.28}
+
+[INFO|callbacks.py:319] 2024-08-29 19:34:54,105 >> {'loss': 0.0078, 'learning_rate': 6.6906e-06, 'epoch': 2.28, 'throughput': 443.17}
+
+[INFO|callbacks.py:319] 2024-08-29 19:35:05,306 >> {'loss': 0.0141, 'learning_rate': 6.1114e-06, 'epoch': 2.32, 'throughput': 442.62}
+
+[INFO|callbacks.py:319] 2024-08-29 19:35:16,627 >> {'loss': 0.0140, 'learning_rate': 5.5550e-06, 'epoch': 2.35, 'throughput': 442.57}
+
+[INFO|callbacks.py:319] 2024-08-29 19:35:27,909 >> {'loss': 0.0045, 'learning_rate': 5.0219e-06, 'epoch': 2.38, 'throughput': 441.94}
+
+[INFO|callbacks.py:319] 2024-08-29 19:35:38,770 >> {'loss': 0.0062, 'learning_rate': 4.5129e-06, 'epoch': 2.42, 'throughput': 441.94}
+
+[INFO|callbacks.py:319] 2024-08-29 19:35:50,161 >> {'loss': 0.0028, 'learning_rate': 4.0285e-06, 'epoch': 2.45, 'throughput': 441.91}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:01,253 >> {'loss': 0.0022, 'learning_rate': 3.5693e-06, 'epoch': 2.48, 'throughput': 441.82}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:11,807 >> {'loss': 0.0061, 'learning_rate': 3.1359e-06, 'epoch': 2.52, 'throughput': 441.87}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:22,359 >> {'loss': 0.0077, 'learning_rate': 2.7288e-06, 'epoch': 2.55, 'throughput': 441.94}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:33,055 >> {'loss': 0.0053, 'learning_rate': 2.3484e-06, 'epoch': 2.58, 'throughput': 441.82}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:43,743 >> {'loss': 0.0021, 'learning_rate': 1.9953e-06, 'epoch': 2.62, 'throughput': 442.19}
+
+[INFO|callbacks.py:319] 2024-08-29 19:36:54,467 >> {'loss': 0.0063, 'learning_rate': 1.6698e-06, 'epoch': 2.65, 'throughput': 442.11}
+
+[INFO|trainer.py:3503] 2024-08-29 19:36:54,467 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-400
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:36:54,690 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:36:54,691 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:36:54,768 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-400/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:36:54,768 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-400/special_tokens_map.json
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:05,578 >> {'loss': 0.0052, 'learning_rate': 1.3724e-06, 'epoch': 2.68, 'throughput': 442.09}
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:16,110 >> {'loss': 0.0051, 'learning_rate': 1.1034e-06, 'epoch': 2.72, 'throughput': 442.47}
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:26,696 >> {'loss': 0.0084, 'learning_rate': 8.6311e-07, 'epoch': 2.75, 'throughput': 442.54}
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:37,213 >> {'loss': 0.0122, 'learning_rate': 6.5185e-07, 'epoch': 2.78, 'throughput': 442.77}
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:48,006 >> {'loss': 0.0060, 'learning_rate': 4.6986e-07, 'epoch': 2.81, 'throughput': 442.71}
+
+[INFO|callbacks.py:319] 2024-08-29 19:37:58,673 >> {'loss': 0.0044, 'learning_rate': 3.1736e-07, 'epoch': 2.85, 'throughput': 442.88}
+
+[INFO|callbacks.py:319] 2024-08-29 19:38:09,686 >> {'loss': 0.0091, 'learning_rate': 1.9453e-07, 'epoch': 2.88, 'throughput': 442.96}
+
+[INFO|callbacks.py:319] 2024-08-29 19:38:20,873 >> {'loss': 0.0050, 'learning_rate': 1.0153e-07, 'epoch': 2.91, 'throughput': 443.07}
+
+[INFO|callbacks.py:319] 2024-08-29 19:38:32,015 >> {'loss': 0.0051, 'learning_rate': 3.8466e-08, 'epoch': 2.95, 'throughput': 443.03}
+
+[INFO|callbacks.py:319] 2024-08-29 19:38:43,205 >> {'loss': 0.0049, 'learning_rate': 5.4105e-09, 'epoch': 2.98, 'throughput': 442.9}
+
+[INFO|trainer.py:3503] 2024-08-29 19:38:49,911 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-453
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:38:50,138 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:38:50,138 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:38:50,212 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-453/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:38:50,212 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/checkpoint-453/special_tokens_map.json
+
+[INFO|trainer.py:2394] 2024-08-29 19:38:50,404 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|trainer.py:3503] 2024-08-29 19:38:50,405 >> Saving model checkpoint to saves/Mistral-7B-v0.1-Chat/lora/mistral_docker
+
+[INFO|configuration_utils.py:733] 2024-08-29 19:38:50,630 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--mistralai--Mistral-7B-Instruct-v0.1/snapshots/2dcff66eac0c01dc50e4c41eea959968232187fe/config.json
+
+[INFO|configuration_utils.py:800] 2024-08-29 19:38:50,631 >> Model config MistralConfig {
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.4",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|tokenization_utils_base.py:2702] 2024-08-29 19:38:50,703 >> tokenizer config file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2711] 2024-08-29 19:38:50,704 >> Special tokens file saved in saves/Mistral-7B-v0.1-Chat/lora/mistral_docker/special_tokens_map.json
+
+[WARNING|ploting.py:89] 2024-08-29 19:38:50,814 >> No metric eval_loss to plot.
+
+[WARNING|ploting.py:89] 2024-08-29 19:38:50,814 >> No metric eval_accuracy to plot.
+
+[INFO|modelcard.py:449] 2024-08-29 19:38:50,816 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,46 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ '<s>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "num_input_tokens_seen": 437216,
+    "total_flos": 1.8708380691726336e+16,
+    "train_loss": 0.044493223137528556,
+    "train_runtime": 988.3228,
+    "train_samples_per_second": 7.331,
+    "train_steps_per_second": 0.458
+}

trainer_log.jsonl

@@ -0,0 +1,91 @@
+{"current_steps": 5, "total_steps": 453, "loss": 0.6997, "learning_rate": 4.998497170031657e-05, "epoch": 0.033112582781456956, "percentage": 1.1, "elapsed_time": "0:00:11", "remaining_time": "0:17:06", "throughput": 428.77, "total_tokens": 4912}
+{"current_steps": 10, "total_steps": 453, "loss": 0.3509, "learning_rate": 4.9939904869249616e-05, "epoch": 0.06622516556291391, "percentage": 2.21, "elapsed_time": "0:00:22", "remaining_time": "0:16:36", "throughput": 441.91, "total_tokens": 9936}
+{"current_steps": 15, "total_steps": 453, "loss": 0.2853, "learning_rate": 4.9864853689026556e-05, "epoch": 0.09933774834437085, "percentage": 3.31, "elapsed_time": "0:00:33", "remaining_time": "0:16:17", "throughput": 443.85, "total_tokens": 14864}
+{"current_steps": 20, "total_steps": 453, "loss": 0.2456, "learning_rate": 4.975990839097764e-05, "epoch": 0.13245033112582782, "percentage": 4.42, "elapsed_time": "0:00:44", "remaining_time": "0:15:56", "throughput": 448.8, "total_tokens": 19824}
+{"current_steps": 25, "total_steps": 453, "loss": 0.2084, "learning_rate": 4.9625195147054034e-05, "epoch": 0.16556291390728478, "percentage": 5.52, "elapsed_time": "0:00:55", "remaining_time": "0:15:43", "throughput": 444.22, "total_tokens": 24480}
+{"current_steps": 30, "total_steps": 453, "loss": 0.1626, "learning_rate": 4.9460875918135804e-05, "epoch": 0.1986754966887417, "percentage": 6.62, "elapsed_time": "0:01:05", "remaining_time": "0:15:30", "throughput": 444.12, "total_tokens": 29296}
+{"current_steps": 35, "total_steps": 453, "loss": 0.1405, "learning_rate": 4.9267148259312224e-05, "epoch": 0.23178807947019867, "percentage": 7.73, "elapsed_time": "0:01:16", "remaining_time": "0:15:15", "throughput": 442.72, "total_tokens": 33936}
+{"current_steps": 40, "total_steps": 453, "loss": 0.1182, "learning_rate": 4.9044245082368415e-05, "epoch": 0.26490066225165565, "percentage": 8.83, "elapsed_time": "0:01:27", "remaining_time": "0:15:07", "throughput": 444.57, "total_tokens": 39056}
+{"current_steps": 45, "total_steps": 453, "loss": 0.1285, "learning_rate": 4.879243437576383e-05, "epoch": 0.2980132450331126, "percentage": 9.93, "elapsed_time": "0:01:39", "remaining_time": "0:15:00", "throughput": 438.23, "total_tokens": 43520}
+{"current_steps": 50, "total_steps": 453, "loss": 0.0906, "learning_rate": 4.8512018882439475e-05, "epoch": 0.33112582781456956, "percentage": 11.04, "elapsed_time": "0:01:50", "remaining_time": "0:14:50", "throughput": 440.48, "total_tokens": 48656}
+{"current_steps": 55, "total_steps": 453, "loss": 0.054, "learning_rate": 4.820333573584091e-05, "epoch": 0.36423841059602646, "percentage": 12.14, "elapsed_time": "0:02:01", "remaining_time": "0:14:37", "throughput": 441.77, "total_tokens": 53600}
+{"current_steps": 60, "total_steps": 453, "loss": 0.073, "learning_rate": 4.786675605459487e-05, "epoch": 0.3973509933774834, "percentage": 13.25, "elapsed_time": "0:02:12", "remaining_time": "0:14:25", "throughput": 442.07, "total_tokens": 58384}
+{"current_steps": 65, "total_steps": 453, "loss": 0.0524, "learning_rate": 4.7502684496326746e-05, "epoch": 0.4304635761589404, "percentage": 14.35, "elapsed_time": "0:02:22", "remaining_time": "0:14:11", "throughput": 442.81, "total_tokens": 63152}
+{"current_steps": 70, "total_steps": 453, "loss": 0.0534, "learning_rate": 4.711155877115523e-05, "epoch": 0.46357615894039733, "percentage": 15.45, "elapsed_time": "0:02:33", "remaining_time": "0:13:58", "throughput": 442.95, "total_tokens": 67888}
+{"current_steps": 75, "total_steps": 453, "loss": 0.0759, "learning_rate": 4.669384911544927e-05, "epoch": 0.4966887417218543, "percentage": 16.56, "elapsed_time": "0:02:43", "remaining_time": "0:13:46", "throughput": 442.11, "total_tokens": 72480}
+{"current_steps": 80, "total_steps": 453, "loss": 0.0766, "learning_rate": 4.625005772647979e-05, "epoch": 0.5298013245033113, "percentage": 17.66, "elapsed_time": "0:02:54", "remaining_time": "0:13:35", "throughput": 440.9, "total_tokens": 77120}
+{"current_steps": 85, "total_steps": 453, "loss": 0.0383, "learning_rate": 4.578071815864602e-05, "epoch": 0.5629139072847682, "percentage": 18.76, "elapsed_time": "0:03:05", "remaining_time": "0:13:25", "throughput": 438.74, "total_tokens": 81584}
+{"current_steps": 90, "total_steps": 453, "loss": 0.0393, "learning_rate": 4.528639468200226e-05, "epoch": 0.5960264900662252, "percentage": 19.87, "elapsed_time": "0:03:16", "remaining_time": "0:13:13", "throughput": 439.35, "total_tokens": 86416}
+{"current_steps": 95, "total_steps": 453, "loss": 0.0589, "learning_rate": 4.476768160385632e-05, "epoch": 0.6291390728476821, "percentage": 20.97, "elapsed_time": "0:03:27", "remaining_time": "0:13:00", "throughput": 440.4, "total_tokens": 91248}
+{"current_steps": 100, "total_steps": 453, "loss": 0.0351, "learning_rate": 4.4225202554255227e-05, "epoch": 0.6622516556291391, "percentage": 22.08, "elapsed_time": "0:03:37", "remaining_time": "0:12:49", "throughput": 440.1, "total_tokens": 95936}
+{"current_steps": 105, "total_steps": 453, "loss": 0.0417, "learning_rate": 4.3659609736217344e-05, "epoch": 0.695364238410596, "percentage": 23.18, "elapsed_time": "0:03:49", "remaining_time": "0:12:39", "throughput": 439.61, "total_tokens": 100704}
+{"current_steps": 110, "total_steps": 453, "loss": 0.0437, "learning_rate": 4.3071583141612135e-05, "epoch": 0.7284768211920529, "percentage": 24.28, "elapsed_time": "0:03:59", "remaining_time": "0:12:28", "throughput": 439.17, "total_tokens": 105376}
+{"current_steps": 115, "total_steps": 453, "loss": 0.0498, "learning_rate": 4.2461829733630435e-05, "epoch": 0.7615894039735099, "percentage": 25.39, "elapsed_time": "0:04:10", "remaining_time": "0:12:17", "throughput": 439.15, "total_tokens": 110208}
+{"current_steps": 120, "total_steps": 453, "loss": 0.0239, "learning_rate": 4.1831082596828106e-05, "epoch": 0.7947019867549668, "percentage": 26.49, "elapsed_time": "0:04:21", "remaining_time": "0:12:06", "throughput": 438.31, "total_tokens": 114704}
+{"current_steps": 125, "total_steps": 453, "loss": 0.0228, "learning_rate": 4.118010005576485e-05, "epoch": 0.8278145695364238, "percentage": 27.59, "elapsed_time": "0:04:32", "remaining_time": "0:11:54", "throughput": 439.58, "total_tokens": 119744}
+{"current_steps": 130, "total_steps": 453, "loss": 0.039, "learning_rate": 4.050966476329793e-05, "epoch": 0.8609271523178808, "percentage": 28.7, "elapsed_time": "0:04:43", "remaining_time": "0:11:44", "throughput": 440.13, "total_tokens": 124736}
+{"current_steps": 135, "total_steps": 453, "loss": 0.05, "learning_rate": 3.9820582759626825e-05, "epoch": 0.8940397350993378, "percentage": 29.8, "elapsed_time": "0:04:54", "remaining_time": "0:11:32", "throughput": 440.36, "total_tokens": 129552}
+{"current_steps": 140, "total_steps": 453, "loss": 0.02, "learning_rate": 3.911368250322014e-05, "epoch": 0.9271523178807947, "percentage": 30.91, "elapsed_time": "0:05:04", "remaining_time": "0:11:21", "throughput": 441.12, "total_tokens": 134400}
+{"current_steps": 145, "total_steps": 453, "loss": 0.0159, "learning_rate": 3.8389813874789856e-05, "epoch": 0.9602649006622517, "percentage": 32.01, "elapsed_time": "0:05:15", "remaining_time": "0:11:10", "throughput": 441.95, "total_tokens": 139424}
+{"current_steps": 150, "total_steps": 453, "loss": 0.017, "learning_rate": 3.764984715551032e-05, "epoch": 0.9933774834437086, "percentage": 33.11, "elapsed_time": "0:05:25", "remaining_time": "0:10:58", "throughput": 442.86, "total_tokens": 144368}
+{"current_steps": 155, "total_steps": 453, "loss": 0.0298, "learning_rate": 3.6894671980710574e-05, "epoch": 1.0264900662251655, "percentage": 34.22, "elapsed_time": "0:05:36", "remaining_time": "0:10:46", "throughput": 443.06, "total_tokens": 149040}
+{"current_steps": 160, "total_steps": 453, "loss": 0.0357, "learning_rate": 3.612519627029787e-05, "epoch": 1.0596026490066226, "percentage": 35.32, "elapsed_time": "0:05:46", "remaining_time": "0:10:35", "throughput": 443.05, "total_tokens": 153712}
+{"current_steps": 165, "total_steps": 453, "loss": 0.0185, "learning_rate": 3.534234513719821e-05, "epoch": 1.0927152317880795, "percentage": 36.42, "elapsed_time": "0:05:57", "remaining_time": "0:10:23", "throughput": 444.01, "total_tokens": 158640}
+{"current_steps": 170, "total_steps": 453, "loss": 0.0339, "learning_rate": 3.4547059775126445e-05, "epoch": 1.1258278145695364, "percentage": 37.53, "elapsed_time": "0:06:08", "remaining_time": "0:10:12", "throughput": 444.2, "total_tokens": 163552}
+{"current_steps": 175, "total_steps": 453, "loss": 0.0224, "learning_rate": 3.3740296327022984e-05, "epoch": 1.1589403973509933, "percentage": 38.63, "elapsed_time": "0:06:19", "remaining_time": "0:10:02", "throughput": 444.02, "total_tokens": 168352}
+{"current_steps": 180, "total_steps": 453, "loss": 0.0192, "learning_rate": 3.292302473551757e-05, "epoch": 1.1920529801324504, "percentage": 39.74, "elapsed_time": "0:06:29", "remaining_time": "0:09:51", "throughput": 444.69, "total_tokens": 173312}
+{"current_steps": 185, "total_steps": 453, "loss": 0.0275, "learning_rate": 3.20962275768022e-05, "epoch": 1.2251655629139073, "percentage": 40.84, "elapsed_time": "0:06:40", "remaining_time": "0:09:40", "throughput": 444.69, "total_tokens": 178160}
+{"current_steps": 190, "total_steps": 453, "loss": 0.0086, "learning_rate": 3.126089887931515e-05, "epoch": 1.2582781456953642, "percentage": 41.94, "elapsed_time": "0:06:51", "remaining_time": "0:09:29", "throughput": 444.59, "total_tokens": 182944}
+{"current_steps": 195, "total_steps": 453, "loss": 0.0166, "learning_rate": 3.0418042928656414e-05, "epoch": 1.2913907284768211, "percentage": 43.05, "elapsed_time": "0:07:02", "remaining_time": "0:09:18", "throughput": 445.07, "total_tokens": 187920}
+{"current_steps": 200, "total_steps": 453, "loss": 0.0167, "learning_rate": 2.9568673060171326e-05, "epoch": 1.3245033112582782, "percentage": 44.15, "elapsed_time": "0:07:13", "remaining_time": "0:09:08", "throughput": 445.5, "total_tokens": 192992}
+{"current_steps": 205, "total_steps": 453, "loss": 0.0137, "learning_rate": 2.8713810440653926e-05, "epoch": 1.3576158940397351, "percentage": 45.25, "elapsed_time": "0:07:24", "remaining_time": "0:08:57", "throughput": 444.49, "total_tokens": 197616}
+{"current_steps": 210, "total_steps": 453, "loss": 0.0142, "learning_rate": 2.7854482840634965e-05, "epoch": 1.390728476821192, "percentage": 46.36, "elapsed_time": "0:07:35", "remaining_time": "0:08:46", "throughput": 444.44, "total_tokens": 202400}
+{"current_steps": 215, "total_steps": 453, "loss": 0.0136, "learning_rate": 2.6991723398730383e-05, "epoch": 1.423841059602649, "percentage": 47.46, "elapsed_time": "0:07:46", "remaining_time": "0:08:36", "throughput": 444.3, "total_tokens": 207216}
+{"current_steps": 220, "total_steps": 453, "loss": 0.0184, "learning_rate": 2.6126569379535985e-05, "epoch": 1.4569536423841059, "percentage": 48.57, "elapsed_time": "0:07:57", "remaining_time": "0:08:25", "throughput": 443.79, "total_tokens": 211744}
+{"current_steps": 225, "total_steps": 453, "loss": 0.0098, "learning_rate": 2.526006092656161e-05, "epoch": 1.490066225165563, "percentage": 49.67, "elapsed_time": "0:08:08", "remaining_time": "0:08:14", "throughput": 443.82, "total_tokens": 216608}
+{"current_steps": 230, "total_steps": 453, "loss": 0.012, "learning_rate": 2.4393239811704e-05, "epoch": 1.5231788079470199, "percentage": 50.77, "elapsed_time": "0:08:18", "remaining_time": "0:08:03", "throughput": 444.28, "total_tokens": 221552}
+{"current_steps": 235, "total_steps": 453, "loss": 0.0106, "learning_rate": 2.3527148182762054e-05, "epoch": 1.5562913907284768, "percentage": 51.88, "elapsed_time": "0:08:29", "remaining_time": "0:07:52", "throughput": 443.85, "total_tokens": 226272}
+{"current_steps": 240, "total_steps": 453, "loss": 0.0128, "learning_rate": 2.2662827310499995e-05, "epoch": 1.589403973509934, "percentage": 52.98, "elapsed_time": "0:08:40", "remaining_time": "0:07:42", "throughput": 443.72, "total_tokens": 231072}
+{"current_steps": 245, "total_steps": 453, "loss": 0.0139, "learning_rate": 2.1801316336765126e-05, "epoch": 1.6225165562913908, "percentage": 54.08, "elapsed_time": "0:08:51", "remaining_time": "0:07:31", "throughput": 443.33, "total_tokens": 235728}
+{"current_steps": 250, "total_steps": 453, "loss": 0.0084, "learning_rate": 2.0943651025164932e-05, "epoch": 1.6556291390728477, "percentage": 55.19, "elapsed_time": "0:09:02", "remaining_time": "0:07:20", "throughput": 443.48, "total_tokens": 240560}
+{"current_steps": 255, "total_steps": 453, "loss": 0.0143, "learning_rate": 2.0090862515805898e-05, "epoch": 1.6887417218543046, "percentage": 56.29, "elapsed_time": "0:09:13", "remaining_time": "0:07:09", "throughput": 443.53, "total_tokens": 245408}
+{"current_steps": 260, "total_steps": 453, "loss": 0.011, "learning_rate": 1.9243976085590824e-05, "epoch": 1.7218543046357615, "percentage": 57.4, "elapsed_time": "0:09:24", "remaining_time": "0:06:58", "throughput": 443.94, "total_tokens": 250400}
+{"current_steps": 265, "total_steps": 453, "loss": 0.0127, "learning_rate": 1.840400991556541e-05, "epoch": 1.7549668874172184, "percentage": 58.5, "elapsed_time": "0:09:35", "remaining_time": "0:06:48", "throughput": 443.72, "total_tokens": 255216}
+{"current_steps": 270, "total_steps": 453, "loss": 0.0127, "learning_rate": 1.7571973866795815e-05, "epoch": 1.7880794701986755, "percentage": 59.6, "elapsed_time": "0:09:45", "remaining_time": "0:06:37", "throughput": 443.93, "total_tokens": 260080}
+{"current_steps": 275, "total_steps": 453, "loss": 0.0122, "learning_rate": 1.6748868266249114e-05, "epoch": 1.8211920529801324, "percentage": 60.71, "elapsed_time": "0:09:56", "remaining_time": "0:06:26", "throughput": 443.72, "total_tokens": 264848}
+{"current_steps": 280, "total_steps": 453, "loss": 0.0169, "learning_rate": 1.5935682704136183e-05, "epoch": 1.8543046357615895, "percentage": 61.81, "elapsed_time": "0:10:07", "remaining_time": "0:06:15", "throughput": 443.77, "total_tokens": 269776}
+{"current_steps": 285, "total_steps": 453, "loss": 0.0242, "learning_rate": 1.5133394844163093e-05, "epoch": 1.8874172185430464, "percentage": 62.91, "elapsed_time": "0:10:18", "remaining_time": "0:06:04", "throughput": 444.05, "total_tokens": 274752}
+{"current_steps": 290, "total_steps": 453, "loss": 0.0079, "learning_rate": 1.4342969248121185e-05, "epoch": 1.9205298013245033, "percentage": 64.02, "elapsed_time": "0:10:29", "remaining_time": "0:05:53", "throughput": 443.8, "total_tokens": 279440}
+{"current_steps": 295, "total_steps": 453, "loss": 0.0194, "learning_rate": 1.3565356216229268e-05, "epoch": 1.9536423841059603, "percentage": 65.12, "elapsed_time": "0:10:41", "remaining_time": "0:05:43", "throughput": 443.36, "total_tokens": 284288}
+{"current_steps": 300, "total_steps": 453, "loss": 0.0133, "learning_rate": 1.2801490644621789e-05, "epoch": 1.9867549668874172, "percentage": 66.23, "elapsed_time": "0:10:52", "remaining_time": "0:05:32", "throughput": 443.09, "total_tokens": 289216}
+{"current_steps": 305, "total_steps": 453, "loss": 0.0049, "learning_rate": 1.2052290901357025e-05, "epoch": 2.019867549668874, "percentage": 67.33, "elapsed_time": "0:11:04", "remaining_time": "0:05:22", "throughput": 442.42, "total_tokens": 293992}
+{"current_steps": 310, "total_steps": 453, "loss": 0.0121, "learning_rate": 1.1318657722296097e-05, "epoch": 2.052980132450331, "percentage": 68.43, "elapsed_time": "0:11:15", "remaining_time": "0:05:11", "throughput": 442.9, "total_tokens": 299048}
+{"current_steps": 315, "total_steps": 453, "loss": 0.0102, "learning_rate": 1.0601473128180855e-05, "epoch": 2.0860927152317883, "percentage": 69.54, "elapsed_time": "0:11:25", "remaining_time": "0:05:00", "throughput": 442.89, "total_tokens": 303768}
+{"current_steps": 320, "total_steps": 453, "loss": 0.0072, "learning_rate": 9.90159936421197e-06, "epoch": 2.119205298013245, "percentage": 70.64, "elapsed_time": "0:11:37", "remaining_time": "0:04:49", "throughput": 443.13, "total_tokens": 308872}
+{"current_steps": 325, "total_steps": 453, "loss": 0.0066, "learning_rate": 9.219877863402682e-06, "epoch": 2.152317880794702, "percentage": 71.74, "elapsed_time": "0:11:48", "remaining_time": "0:04:38", "throughput": 443.2, "total_tokens": 313800}
+{"current_steps": 330, "total_steps": 453, "loss": 0.0119, "learning_rate": 8.55712823495419e-06, "epoch": 2.185430463576159, "percentage": 72.85, "elapsed_time": "0:11:59", "remaining_time": "0:04:28", "throughput": 442.86, "total_tokens": 318536}
+{"current_steps": 335, "total_steps": 453, "loss": 0.0057, "learning_rate": 7.91414727886898e-06, "epoch": 2.218543046357616, "percentage": 73.95, "elapsed_time": "0:12:10", "remaining_time": "0:04:17", "throughput": 442.91, "total_tokens": 323512}
+{"current_steps": 340, "total_steps": 453, "loss": 0.0136, "learning_rate": 7.291708027986988e-06, "epoch": 2.251655629139073, "percentage": 75.06, "elapsed_time": "0:12:21", "remaining_time": "0:04:06", "throughput": 443.28, "total_tokens": 328552}
+{"current_steps": 345, "total_steps": 453, "loss": 0.0078, "learning_rate": 6.690558818595943e-06, "epoch": 2.2847682119205297, "percentage": 76.16, "elapsed_time": "0:12:32", "remaining_time": "0:03:55", "throughput": 443.17, "total_tokens": 333272}
+{"current_steps": 350, "total_steps": 453, "loss": 0.0141, "learning_rate": 6.111422390733715e-06, "epoch": 2.3178807947019866, "percentage": 77.26, "elapsed_time": "0:12:43", "remaining_time": "0:03:44", "throughput": 442.62, "total_tokens": 337816}
+{"current_steps": 355, "total_steps": 453, "loss": 0.014, "learning_rate": 5.55499501926394e-06, "epoch": 2.3509933774834435, "percentage": 78.37, "elapsed_time": "0:12:54", "remaining_time": "0:03:33", "throughput": 442.57, "total_tokens": 342792}
+{"current_steps": 360, "total_steps": 453, "loss": 0.0045, "learning_rate": 5.02194567676986e-06, "epoch": 2.384105960264901, "percentage": 79.47, "elapsed_time": "0:13:05", "remaining_time": "0:03:23", "throughput": 441.94, "total_tokens": 347288}
+{"current_steps": 365, "total_steps": 453, "loss": 0.0062, "learning_rate": 4.51291522927268e-06, "epoch": 2.4172185430463577, "percentage": 80.57, "elapsed_time": "0:13:16", "remaining_time": "0:03:12", "throughput": 441.94, "total_tokens": 352088}
+{"current_steps": 370, "total_steps": 453, "loss": 0.0028, "learning_rate": 4.028515665741439e-06, "epoch": 2.4503311258278146, "percentage": 81.68, "elapsed_time": "0:13:28", "remaining_time": "0:03:01", "throughput": 441.91, "total_tokens": 357096}
+{"current_steps": 375, "total_steps": 453, "loss": 0.0022, "learning_rate": 3.5693293623207086e-06, "epoch": 2.4834437086092715, "percentage": 82.78, "elapsed_time": "0:13:39", "remaining_time": "0:02:50", "throughput": 441.82, "total_tokens": 361928}
+{"current_steps": 380, "total_steps": 453, "loss": 0.0061, "learning_rate": 3.135908382160771e-06, "epoch": 2.5165562913907285, "percentage": 83.89, "elapsed_time": "0:13:49", "remaining_time": "0:02:39", "throughput": 441.87, "total_tokens": 366632}
+{"current_steps": 385, "total_steps": 453, "loss": 0.0077, "learning_rate": 2.728773811691923e-06, "epoch": 2.5496688741721854, "percentage": 84.99, "elapsed_time": "0:14:00", "remaining_time": "0:02:28", "throughput": 441.94, "total_tokens": 371352}
+{"current_steps": 390, "total_steps": 453, "loss": 0.0053, "learning_rate": 2.348415134141102e-06, "epoch": 2.5827814569536423, "percentage": 86.09, "elapsed_time": "0:14:10", "remaining_time": "0:02:17", "throughput": 441.82, "total_tokens": 375976}
+{"current_steps": 395, "total_steps": 453, "loss": 0.0021, "learning_rate": 1.995289641043768e-06, "epoch": 2.6158940397350996, "percentage": 87.2, "elapsed_time": "0:14:21", "remaining_time": "0:02:06", "throughput": 442.19, "total_tokens": 381016}
+{"current_steps": 400, "total_steps": 453, "loss": 0.0063, "learning_rate": 1.6698218824588164e-06, "epoch": 2.6490066225165565, "percentage": 88.3, "elapsed_time": "0:14:32", "remaining_time": "0:01:55", "throughput": 442.11, "total_tokens": 385688}
+{"current_steps": 405, "total_steps": 453, "loss": 0.0052, "learning_rate": 1.3724031565473112e-06, "epoch": 2.6821192052980134, "percentage": 89.4, "elapsed_time": "0:14:43", "remaining_time": "0:01:44", "throughput": 442.09, "total_tokens": 390584}
+{"current_steps": 410, "total_steps": 453, "loss": 0.0051, "learning_rate": 1.1033910391288065e-06, "epoch": 2.7152317880794703, "percentage": 90.51, "elapsed_time": "0:14:54", "remaining_time": "0:01:33", "throughput": 442.47, "total_tokens": 395576}
+{"current_steps": 415, "total_steps": 453, "loss": 0.0084, "learning_rate": 8.631089537808307e-07, "epoch": 2.748344370860927, "percentage": 91.61, "elapsed_time": "0:15:04", "remaining_time": "0:01:22", "throughput": 442.54, "total_tokens": 400328}
+{"current_steps": 420, "total_steps": 453, "loss": 0.0122, "learning_rate": 6.51845782998356e-07, "epoch": 2.781456953642384, "percentage": 92.72, "elapsed_time": "0:15:15", "remaining_time": "0:01:11", "throughput": 442.77, "total_tokens": 405192}
+{"current_steps": 425, "total_steps": 453, "loss": 0.006, "learning_rate": 4.698555208807853e-07, "epoch": 2.814569536423841, "percentage": 93.82, "elapsed_time": "0:15:25", "remaining_time": "0:01:01", "throughput": 442.71, "total_tokens": 409912}
+{"current_steps": 430, "total_steps": 453, "loss": 0.0044, "learning_rate": 3.1735696776400703e-07, "epoch": 2.847682119205298, "percentage": 94.92, "elapsed_time": "0:15:36", "remaining_time": "0:00:50", "throughput": 442.88, "total_tokens": 414792}
+{"current_steps": 435, "total_steps": 453, "loss": 0.0091, "learning_rate": 1.9453346716462317e-07, "epoch": 2.880794701986755, "percentage": 96.03, "elapsed_time": "0:15:47", "remaining_time": "0:00:39", "throughput": 442.96, "total_tokens": 419752}
+{"current_steps": 440, "total_steps": 453, "loss": 0.005, "learning_rate": 1.0153268535264827e-07, "epoch": 2.9139072847682117, "percentage": 97.13, "elapsed_time": "0:15:58", "remaining_time": "0:00:28", "throughput": 443.07, "total_tokens": 424808}
+{"current_steps": 445, "total_steps": 453, "loss": 0.0051, "learning_rate": 3.846643381766879e-08, "epoch": 2.9470198675496686, "percentage": 98.23, "elapsed_time": "0:16:09", "remaining_time": "0:00:17", "throughput": 443.03, "total_tokens": 429704}
+{"current_steps": 450, "total_steps": 453, "loss": 0.0049, "learning_rate": 5.41053484192644e-09, "epoch": 2.980132450331126, "percentage": 99.34, "elapsed_time": "0:16:21", "remaining_time": "0:00:06", "throughput": 442.9, "total_tokens": 434536}
+{"current_steps": 453, "total_steps": 453, "epoch": 3.0, "percentage": 100.0, "elapsed_time": "0:16:28", "remaining_time": "0:00:00", "throughput": 442.38, "total_tokens": 437216}

trainer_state.json

@@ -0,0 +1,763 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 453,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.033112582781456956,
+      "grad_norm": 2.2606041431427,
+      "learning_rate": 4.998497170031657e-05,
+      "loss": 0.6997,
+      "num_input_tokens_seen": 4912,
+      "step": 5
+    },
+    {
+      "epoch": 0.06622516556291391,
+      "grad_norm": 2.3543715476989746,
+      "learning_rate": 4.9939904869249616e-05,
+      "loss": 0.3509,
+      "num_input_tokens_seen": 9936,
+      "step": 10
+    },
+    {
+      "epoch": 0.09933774834437085,
+      "grad_norm": 1.801797866821289,
+      "learning_rate": 4.9864853689026556e-05,
+      "loss": 0.2853,
+      "num_input_tokens_seen": 14864,
+      "step": 15
+    },
+    {
+      "epoch": 0.13245033112582782,
+      "grad_norm": 1.3575855493545532,
+      "learning_rate": 4.975990839097764e-05,
+      "loss": 0.2456,
+      "num_input_tokens_seen": 19824,
+      "step": 20
+    },
+    {
+      "epoch": 0.16556291390728478,
+      "grad_norm": 2.5929694175720215,
+      "learning_rate": 4.9625195147054034e-05,
+      "loss": 0.2084,
+      "num_input_tokens_seen": 24480,
+      "step": 25
+    },
+    {
+      "epoch": 0.1986754966887417,
+      "grad_norm": 0.8744802474975586,
+      "learning_rate": 4.9460875918135804e-05,
+      "loss": 0.1626,
+      "num_input_tokens_seen": 29296,
+      "step": 30
+    },
+    {
+      "epoch": 0.23178807947019867,
+      "grad_norm": 2.857471466064453,
+      "learning_rate": 4.9267148259312224e-05,
+      "loss": 0.1405,
+      "num_input_tokens_seen": 33936,
+      "step": 35
+    },
+    {
+      "epoch": 0.26490066225165565,
+      "grad_norm": 1.7763206958770752,
+      "learning_rate": 4.9044245082368415e-05,
+      "loss": 0.1182,
+      "num_input_tokens_seen": 39056,
+      "step": 40
+    },
+    {
+      "epoch": 0.2980132450331126,
+      "grad_norm": 2.78945255279541,
+      "learning_rate": 4.879243437576383e-05,
+      "loss": 0.1285,
+      "num_input_tokens_seen": 43520,
+      "step": 45
+    },
+    {
+      "epoch": 0.33112582781456956,
+      "grad_norm": 1.1319329738616943,
+      "learning_rate": 4.8512018882439475e-05,
+      "loss": 0.0906,
+      "num_input_tokens_seen": 48656,
+      "step": 50
+    },
+    {
+      "epoch": 0.36423841059602646,
+      "grad_norm": 0.8884170055389404,
+      "learning_rate": 4.820333573584091e-05,
+      "loss": 0.054,
+      "num_input_tokens_seen": 53600,
+      "step": 55
+    },
+    {
+      "epoch": 0.3973509933774834,
+      "grad_norm": 2.0057179927825928,
+      "learning_rate": 4.786675605459487e-05,
+      "loss": 0.073,
+      "num_input_tokens_seen": 58384,
+      "step": 60
+    },
+    {
+      "epoch": 0.4304635761589404,
+      "grad_norm": 1.3076105117797852,
+      "learning_rate": 4.7502684496326746e-05,
+      "loss": 0.0524,
+      "num_input_tokens_seen": 63152,
+      "step": 65
+    },
+    {
+      "epoch": 0.46357615894039733,
+      "grad_norm": 1.074171543121338,
+      "learning_rate": 4.711155877115523e-05,
+      "loss": 0.0534,
+      "num_input_tokens_seen": 67888,
+      "step": 70
+    },
+    {
+      "epoch": 0.4966887417218543,
+      "grad_norm": 1.634122371673584,
+      "learning_rate": 4.669384911544927e-05,
+      "loss": 0.0759,
+      "num_input_tokens_seen": 72480,
+      "step": 75
+    },
+    {
+      "epoch": 0.5298013245033113,
+      "grad_norm": 1.0059384107589722,
+      "learning_rate": 4.625005772647979e-05,
+      "loss": 0.0766,
+      "num_input_tokens_seen": 77120,
+      "step": 80
+    },
+    {
+      "epoch": 0.5629139072847682,
+      "grad_norm": 1.3958277702331543,
+      "learning_rate": 4.578071815864602e-05,
+      "loss": 0.0383,
+      "num_input_tokens_seen": 81584,
+      "step": 85
+    },
+    {
+      "epoch": 0.5960264900662252,
+      "grad_norm": 2.721231698989868,
+      "learning_rate": 4.528639468200226e-05,
+      "loss": 0.0393,
+      "num_input_tokens_seen": 86416,
+      "step": 90
+    },
+    {
+      "epoch": 0.6291390728476821,
+      "grad_norm": 1.4776098728179932,
+      "learning_rate": 4.476768160385632e-05,
+      "loss": 0.0589,
+      "num_input_tokens_seen": 91248,
+      "step": 95
+    },
+    {
+      "epoch": 0.6622516556291391,
+      "grad_norm": 1.1105955839157104,
+      "learning_rate": 4.4225202554255227e-05,
+      "loss": 0.0351,
+      "num_input_tokens_seen": 95936,
+      "step": 100
+    },
+    {
+      "epoch": 0.695364238410596,
+      "grad_norm": 1.1301405429840088,
+      "learning_rate": 4.3659609736217344e-05,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 100704,
+      "step": 105
+    },
+    {
+      "epoch": 0.7284768211920529,
+      "grad_norm": 1.869552493095398,
+      "learning_rate": 4.3071583141612135e-05,
+      "loss": 0.0437,
+      "num_input_tokens_seen": 105376,
+      "step": 110
+    },
+    {
+      "epoch": 0.7615894039735099,
+      "grad_norm": 1.0497004985809326,
+      "learning_rate": 4.2461829733630435e-05,
+      "loss": 0.0498,
+      "num_input_tokens_seen": 110208,
+      "step": 115
+    },
+    {
+      "epoch": 0.7947019867549668,
+      "grad_norm": 0.4084171652793884,
+      "learning_rate": 4.1831082596828106e-05,
+      "loss": 0.0239,
+      "num_input_tokens_seen": 114704,
+      "step": 120
+    },
+    {
+      "epoch": 0.8278145695364238,
+      "grad_norm": 0.4082948565483093,
+      "learning_rate": 4.118010005576485e-05,
+      "loss": 0.0228,
+      "num_input_tokens_seen": 119744,
+      "step": 125
+    },
+    {
+      "epoch": 0.8609271523178808,
+      "grad_norm": 0.246117502450943,
+      "learning_rate": 4.050966476329793e-05,
+      "loss": 0.039,
+      "num_input_tokens_seen": 124736,
+      "step": 130
+    },
+    {
+      "epoch": 0.8940397350993378,
+      "grad_norm": 0.5470453500747681,
+      "learning_rate": 3.9820582759626825e-05,
+      "loss": 0.05,
+      "num_input_tokens_seen": 129552,
+      "step": 135
+    },
+    {
+      "epoch": 0.9271523178807947,
+      "grad_norm": 0.6097426414489746,
+      "learning_rate": 3.911368250322014e-05,
+      "loss": 0.02,
+      "num_input_tokens_seen": 134400,
+      "step": 140
+    },
+    {
+      "epoch": 0.9602649006622517,
+      "grad_norm": 0.4274413585662842,
+      "learning_rate": 3.8389813874789856e-05,
+      "loss": 0.0159,
+      "num_input_tokens_seen": 139424,
+      "step": 145
+    },
+    {
+      "epoch": 0.9933774834437086,
+      "grad_norm": 0.1949557512998581,
+      "learning_rate": 3.764984715551032e-05,
+      "loss": 0.017,
+      "num_input_tokens_seen": 144368,
+      "step": 150
+    },
+    {
+      "epoch": 1.0264900662251655,
+      "grad_norm": 0.9226788878440857,
+      "learning_rate": 3.6894671980710574e-05,
+      "loss": 0.0298,
+      "num_input_tokens_seen": 149040,
+      "step": 155
+    },
+    {
+      "epoch": 1.0596026490066226,
+      "grad_norm": 1.5730527639389038,
+      "learning_rate": 3.612519627029787e-05,
+      "loss": 0.0357,
+      "num_input_tokens_seen": 153712,
+      "step": 160
+    },
+    {
+      "epoch": 1.0927152317880795,
+      "grad_norm": 0.5750863552093506,
+      "learning_rate": 3.534234513719821e-05,
+      "loss": 0.0185,
+      "num_input_tokens_seen": 158640,
+      "step": 165
+    },
+    {
+      "epoch": 1.1258278145695364,
+      "grad_norm": 0.4966561198234558,
+      "learning_rate": 3.4547059775126445e-05,
+      "loss": 0.0339,
+      "num_input_tokens_seen": 163552,
+      "step": 170
+    },
+    {
+      "epoch": 1.1589403973509933,
+      "grad_norm": 0.6601276993751526,
+      "learning_rate": 3.3740296327022984e-05,
+      "loss": 0.0224,
+      "num_input_tokens_seen": 168352,
+      "step": 175
+    },
+    {
+      "epoch": 1.1920529801324504,
+      "grad_norm": 1.2051963806152344,
+      "learning_rate": 3.292302473551757e-05,
+      "loss": 0.0192,
+      "num_input_tokens_seen": 173312,
+      "step": 180
+    },
+    {
+      "epoch": 1.2251655629139073,
+      "grad_norm": 0.48197436332702637,
+      "learning_rate": 3.20962275768022e-05,
+      "loss": 0.0275,
+      "num_input_tokens_seen": 178160,
+      "step": 185
+    },
+    {
+      "epoch": 1.2582781456953642,
+      "grad_norm": 0.3365241289138794,
+      "learning_rate": 3.126089887931515e-05,
+      "loss": 0.0086,
+      "num_input_tokens_seen": 182944,
+      "step": 190
+    },
+    {
+      "epoch": 1.2913907284768211,
+      "grad_norm": 0.5984787940979004,
+      "learning_rate": 3.0418042928656414e-05,
+      "loss": 0.0166,
+      "num_input_tokens_seen": 187920,
+      "step": 195
+    },
+    {
+      "epoch": 1.3245033112582782,
+      "grad_norm": 0.3351554274559021,
+      "learning_rate": 2.9568673060171326e-05,
+      "loss": 0.0167,
+      "num_input_tokens_seen": 192992,
+      "step": 200
+    },
+    {
+      "epoch": 1.3576158940397351,
+      "grad_norm": 0.21267688274383545,
+      "learning_rate": 2.8713810440653926e-05,
+      "loss": 0.0137,
+      "num_input_tokens_seen": 197616,
+      "step": 205
+    },
+    {
+      "epoch": 1.390728476821192,
+      "grad_norm": 0.17382262647151947,
+      "learning_rate": 2.7854482840634965e-05,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 202400,
+      "step": 210
+    },
+    {
+      "epoch": 1.423841059602649,
+      "grad_norm": 0.552691638469696,
+      "learning_rate": 2.6991723398730383e-05,
+      "loss": 0.0136,
+      "num_input_tokens_seen": 207216,
+      "step": 215
+    },
+    {
+      "epoch": 1.4569536423841059,
+      "grad_norm": 0.4015306532382965,
+      "learning_rate": 2.6126569379535985e-05,
+      "loss": 0.0184,
+      "num_input_tokens_seen": 211744,
+      "step": 220
+    },
+    {
+      "epoch": 1.490066225165563,
+      "grad_norm": 0.20611760020256042,
+      "learning_rate": 2.526006092656161e-05,
+      "loss": 0.0098,
+      "num_input_tokens_seen": 216608,
+      "step": 225
+    },
+    {
+      "epoch": 1.5231788079470199,
+      "grad_norm": 0.3923095762729645,
+      "learning_rate": 2.4393239811704e-05,
+      "loss": 0.012,
+      "num_input_tokens_seen": 221552,
+      "step": 230
+    },
+    {
+      "epoch": 1.5562913907284768,
+      "grad_norm": 0.2283228635787964,
+      "learning_rate": 2.3527148182762054e-05,
+      "loss": 0.0106,
+      "num_input_tokens_seen": 226272,
+      "step": 235
+    },
+    {
+      "epoch": 1.589403973509934,
+      "grad_norm": 1.2066081762313843,
+      "learning_rate": 2.2662827310499995e-05,
+      "loss": 0.0128,
+      "num_input_tokens_seen": 231072,
+      "step": 240
+    },
+    {
+      "epoch": 1.6225165562913908,
+      "grad_norm": 0.39357009530067444,
+      "learning_rate": 2.1801316336765126e-05,
+      "loss": 0.0139,
+      "num_input_tokens_seen": 235728,
+      "step": 245
+    },
+    {
+      "epoch": 1.6556291390728477,
+      "grad_norm": 0.2383226603269577,
+      "learning_rate": 2.0943651025164932e-05,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 240560,
+      "step": 250
+    },
+    {
+      "epoch": 1.6887417218543046,
+      "grad_norm": 1.2794650793075562,
+      "learning_rate": 2.0090862515805898e-05,
+      "loss": 0.0143,
+      "num_input_tokens_seen": 245408,
+      "step": 255
+    },
+    {
+      "epoch": 1.7218543046357615,
+      "grad_norm": 0.5334519147872925,
+      "learning_rate": 1.9243976085590824e-05,
+      "loss": 0.011,
+      "num_input_tokens_seen": 250400,
+      "step": 260
+    },
+    {
+      "epoch": 1.7549668874172184,
+      "grad_norm": 0.32287150621414185,
+      "learning_rate": 1.840400991556541e-05,
+      "loss": 0.0127,
+      "num_input_tokens_seen": 255216,
+      "step": 265
+    },
+    {
+      "epoch": 1.7880794701986755,
+      "grad_norm": 0.7233603596687317,
+      "learning_rate": 1.7571973866795815e-05,
+      "loss": 0.0127,
+      "num_input_tokens_seen": 260080,
+      "step": 270
+    },
+    {
+      "epoch": 1.8211920529801324,
+      "grad_norm": 0.2672172784805298,
+      "learning_rate": 1.6748868266249114e-05,
+      "loss": 0.0122,
+      "num_input_tokens_seen": 264848,
+      "step": 275
+    },
+    {
+      "epoch": 1.8543046357615895,
+      "grad_norm": 0.32350099086761475,
+      "learning_rate": 1.5935682704136183e-05,
+      "loss": 0.0169,
+      "num_input_tokens_seen": 269776,
+      "step": 280
+    },
+    {
+      "epoch": 1.8874172185430464,
+      "grad_norm": 0.26331964135169983,
+      "learning_rate": 1.5133394844163093e-05,
+      "loss": 0.0242,
+      "num_input_tokens_seen": 274752,
+      "step": 285
+    },
+    {
+      "epoch": 1.9205298013245033,
+      "grad_norm": 0.25610876083374023,
+      "learning_rate": 1.4342969248121185e-05,
+      "loss": 0.0079,
+      "num_input_tokens_seen": 279440,
+      "step": 290
+    },
+    {
+      "epoch": 1.9536423841059603,
+      "grad_norm": 1.2221907377243042,
+      "learning_rate": 1.3565356216229268e-05,
+      "loss": 0.0194,
+      "num_input_tokens_seen": 284288,
+      "step": 295
+    },
+    {
+      "epoch": 1.9867549668874172,
+      "grad_norm": 0.46571576595306396,
+      "learning_rate": 1.2801490644621789e-05,
+      "loss": 0.0133,
+      "num_input_tokens_seen": 289216,
+      "step": 300
+    },
+    {
+      "epoch": 2.019867549668874,
+      "grad_norm": 0.06845066696405411,
+      "learning_rate": 1.2052290901357025e-05,
+      "loss": 0.0049,
+      "num_input_tokens_seen": 293992,
+      "step": 305
+    },
+    {
+      "epoch": 2.052980132450331,
+      "grad_norm": 0.3654076159000397,
+      "learning_rate": 1.1318657722296097e-05,
+      "loss": 0.0121,
+      "num_input_tokens_seen": 299048,
+      "step": 310
+    },
+    {
+      "epoch": 2.0860927152317883,
+      "grad_norm": 0.1736595779657364,
+      "learning_rate": 1.0601473128180855e-05,
+      "loss": 0.0102,
+      "num_input_tokens_seen": 303768,
+      "step": 315
+    },
+    {
+      "epoch": 2.119205298013245,
+      "grad_norm": 0.15334352850914001,
+      "learning_rate": 9.90159936421197e-06,
+      "loss": 0.0072,
+      "num_input_tokens_seen": 308872,
+      "step": 320
+    },
+    {
+      "epoch": 2.152317880794702,
+      "grad_norm": 0.15292225778102875,
+      "learning_rate": 9.219877863402682e-06,
+      "loss": 0.0066,
+      "num_input_tokens_seen": 313800,
+      "step": 325
+    },
+    {
+      "epoch": 2.185430463576159,
+      "grad_norm": 0.12052378058433533,
+      "learning_rate": 8.55712823495419e-06,
+      "loss": 0.0119,
+      "num_input_tokens_seen": 318536,
+      "step": 330
+    },
+    {
+      "epoch": 2.218543046357616,
+      "grad_norm": 0.16140712797641754,
+      "learning_rate": 7.91414727886898e-06,
+      "loss": 0.0057,
+      "num_input_tokens_seen": 323512,
+      "step": 335
+    },
+    {
+      "epoch": 2.251655629139073,
+      "grad_norm": 0.49767032265663147,
+      "learning_rate": 7.291708027986988e-06,
+      "loss": 0.0136,
+      "num_input_tokens_seen": 328552,
+      "step": 340
+    },
+    {
+      "epoch": 2.2847682119205297,
+      "grad_norm": 0.10487841814756393,
+      "learning_rate": 6.690558818595943e-06,
+      "loss": 0.0078,
+      "num_input_tokens_seen": 333272,
+      "step": 345
+    },
+    {
+      "epoch": 2.3178807947019866,
+      "grad_norm": 0.2247195988893509,
+      "learning_rate": 6.111422390733715e-06,
+      "loss": 0.0141,
+      "num_input_tokens_seen": 337816,
+      "step": 350
+    },
+    {
+      "epoch": 2.3509933774834435,
+      "grad_norm": 0.15797987580299377,
+      "learning_rate": 5.55499501926394e-06,
+      "loss": 0.014,
+      "num_input_tokens_seen": 342792,
+      "step": 355
+    },
+    {
+      "epoch": 2.384105960264901,
+      "grad_norm": 0.33236005902290344,
+      "learning_rate": 5.02194567676986e-06,
+      "loss": 0.0045,
+      "num_input_tokens_seen": 347288,
+      "step": 360
+    },
+    {
+      "epoch": 2.4172185430463577,
+      "grad_norm": 0.3886609375476837,
+      "learning_rate": 4.51291522927268e-06,
+      "loss": 0.0062,
+      "num_input_tokens_seen": 352088,
+      "step": 365
+    },
+    {
+      "epoch": 2.4503311258278146,
+      "grad_norm": 0.3580506145954132,
+      "learning_rate": 4.028515665741439e-06,
+      "loss": 0.0028,
+      "num_input_tokens_seen": 357096,
+      "step": 370
+    },
+    {
+      "epoch": 2.4834437086092715,
+      "grad_norm": 0.045303523540496826,
+      "learning_rate": 3.5693293623207086e-06,
+      "loss": 0.0022,
+      "num_input_tokens_seen": 361928,
+      "step": 375
+    },
+    {
+      "epoch": 2.5165562913907285,
+      "grad_norm": 0.3307873010635376,
+      "learning_rate": 3.135908382160771e-06,
+      "loss": 0.0061,
+      "num_input_tokens_seen": 366632,
+      "step": 380
+    },
+    {
+      "epoch": 2.5496688741721854,
+      "grad_norm": 0.31569671630859375,
+      "learning_rate": 2.728773811691923e-06,
+      "loss": 0.0077,
+      "num_input_tokens_seen": 371352,
+      "step": 385
+    },
+    {
+      "epoch": 2.5827814569536423,
+      "grad_norm": 0.25220000743865967,
+      "learning_rate": 2.348415134141102e-06,
+      "loss": 0.0053,
+      "num_input_tokens_seen": 375976,
+      "step": 390
+    },
+    {
+      "epoch": 2.6158940397350996,
+      "grad_norm": 0.12679828703403473,
+      "learning_rate": 1.995289641043768e-06,
+      "loss": 0.0021,
+      "num_input_tokens_seen": 381016,
+      "step": 395
+    },
+    {
+      "epoch": 2.6490066225165565,
+      "grad_norm": 0.2858022153377533,
+      "learning_rate": 1.6698218824588164e-06,
+      "loss": 0.0063,
+      "num_input_tokens_seen": 385688,
+      "step": 400
+    },
+    {
+      "epoch": 2.6821192052980134,
+      "grad_norm": 0.3448869287967682,
+      "learning_rate": 1.3724031565473112e-06,
+      "loss": 0.0052,
+      "num_input_tokens_seen": 390584,
+      "step": 405
+    },
+    {
+      "epoch": 2.7152317880794703,
+      "grad_norm": 0.26856377720832825,
+      "learning_rate": 1.1033910391288065e-06,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 395576,
+      "step": 410
+    },
+    {
+      "epoch": 2.748344370860927,
+      "grad_norm": 0.3648998737335205,
+      "learning_rate": 8.631089537808307e-07,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 400328,
+      "step": 415
+    },
+    {
+      "epoch": 2.781456953642384,
+      "grad_norm": 1.5257205963134766,
+      "learning_rate": 6.51845782998356e-07,
+      "loss": 0.0122,
+      "num_input_tokens_seen": 405192,
+      "step": 420
+    },
+    {
+      "epoch": 2.814569536423841,
+      "grad_norm": 0.4184490144252777,
+      "learning_rate": 4.698555208807853e-07,
+      "loss": 0.006,
+      "num_input_tokens_seen": 409912,
+      "step": 425
+    },
+    {
+      "epoch": 2.847682119205298,
+      "grad_norm": 0.39113691449165344,
+      "learning_rate": 3.1735696776400703e-07,
+      "loss": 0.0044,
+      "num_input_tokens_seen": 414792,
+      "step": 430
+    },
+    {
+      "epoch": 2.880794701986755,
+      "grad_norm": 0.49995261430740356,
+      "learning_rate": 1.9453346716462317e-07,
+      "loss": 0.0091,
+      "num_input_tokens_seen": 419752,
+      "step": 435
+    },
+    {
+      "epoch": 2.9139072847682117,
+      "grad_norm": 0.45024681091308594,
+      "learning_rate": 1.0153268535264827e-07,
+      "loss": 0.005,
+      "num_input_tokens_seen": 424808,
+      "step": 440
+    },
+    {
+      "epoch": 2.9470198675496686,
+      "grad_norm": 0.11354193091392517,
+      "learning_rate": 3.846643381766879e-08,
+      "loss": 0.0051,
+      "num_input_tokens_seen": 429704,
+      "step": 445
+    },
+    {
+      "epoch": 2.980132450331126,
+      "grad_norm": 0.07164571434259415,
+      "learning_rate": 5.41053484192644e-09,
+      "loss": 0.0049,
+      "num_input_tokens_seen": 434536,
+      "step": 450
+    },
+    {
+      "epoch": 3.0,
+      "num_input_tokens_seen": 437216,
+      "step": 453,
+      "total_flos": 1.8708380691726336e+16,
+      "train_loss": 0.044493223137528556,
+      "train_runtime": 988.3228,
+      "train_samples_per_second": 7.331,
+      "train_steps_per_second": 0.458
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 453,
+  "num_input_tokens_seen": 437216,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8708380691726336e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddac98dc4cdbbb7bedc33599ee666b2ce8e71e9f161cc8201762f26175bbe62e
+size 5368

training_args.yaml

@@ -0,0 +1,32 @@
+bf16: true
+cutoff_len: 1024
+dataset: minecraft
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+finetuning_type: lora
+flash_attn: auto
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 5.0e-05
+logging_steps: 5
+lora_alpha: 16
+lora_dropout: 0
+lora_rank: 8
+lora_target: all
+lr_scheduler_type: cosine
+max_grad_norm: 1.0
+max_samples: 100000
+model_name_or_path: mistralai/Mistral-7B-Instruct-v0.1
+num_train_epochs: 3.0
+optim: adamw_torch
+output_dir: saves/Mistral-7B-v0.1-Chat/lora/mistral_docker
+packing: false
+per_device_train_batch_size: 2
+plot_loss: true
+preprocessing_num_workers: 16
+report_to: none
+save_steps: 100
+stage: sft
+template: mistral
+warmup_steps: 0