Training in progress, step 3000, checkpoint

checkpoint-3000/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

checkpoint-3000/adapter_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "EleutherAI/polyglot-ko-1.3b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "query_key_value"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-3000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df84dc4351dcb566c0430da930f2819fda181ce0324d0d821e643857e863f08e
+size 12600958

checkpoint-3000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b42bd87763db6f8f3d40fb9c1e61d9d6d814266a8387da654c8b723af4c83077
+size 25206586

checkpoint-3000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1aa342c2c990fe4b9f5ad99bd0c7b168a6ecd92d91eaf231fa5456f400597bad
+size 14308

checkpoint-3000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04a3ba4380d1fc9c8c37fea4bda3e741ca073b82e1aafed86584a6b99dca028c
+size 1064

checkpoint-3000/special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-3000/tokenizer_config.json

@@ -0,0 +1,72 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|unused0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|unused1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30000": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30001": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30002": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-3000/trainer_state.json

@@ -0,0 +1,103 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.7915057915057915,
+  "eval_steps": 1000,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.239382239382239e-07,
+      "loss": 2.3335,
+      "step": 300
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 1.4478764478764478e-06,
+      "loss": 2.322,
+      "step": 600
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.171814671814672e-06,
+      "loss": 2.268,
+      "step": 900
+    },
+    {
+      "epoch": 1.93,
+      "eval_loss": 2.2363617420196533,
+      "eval_runtime": 56.2328,
+      "eval_samples_per_second": 51.98,
+      "eval_steps_per_second": 3.254,
+      "step": 1000
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 2.8957528957528956e-06,
+      "loss": 2.2427,
+      "step": 1200
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 3.61969111969112e-06,
+      "loss": 2.2085,
+      "step": 1500
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 4.343629343629344e-06,
+      "loss": 2.1902,
+      "step": 1800
+    },
+    {
+      "epoch": 3.86,
+      "eval_loss": 2.1604719161987305,
+      "eval_runtime": 56.4281,
+      "eval_samples_per_second": 51.8,
+      "eval_steps_per_second": 3.243,
+      "step": 2000
+    },
+    {
+      "epoch": 4.05,
+      "learning_rate": 5.067567567567568e-06,
+      "loss": 2.173,
+      "step": 2100
+    },
+    {
+      "epoch": 4.63,
+      "learning_rate": 5.791505791505791e-06,
+      "loss": 2.1517,
+      "step": 2400
+    },
+    {
+      "epoch": 5.21,
+      "learning_rate": 6.515444015444016e-06,
+      "loss": 2.135,
+      "step": 2700
+    },
+    {
+      "epoch": 5.79,
+      "learning_rate": 7.23938223938224e-06,
+      "loss": 2.1156,
+      "step": 3000
+    },
+    {
+      "epoch": 5.79,
+      "eval_loss": 2.108267068862915,
+      "eval_runtime": 56.4468,
+      "eval_samples_per_second": 51.783,
+      "eval_steps_per_second": 3.242,
+      "step": 3000
+    }
+  ],
+  "logging_steps": 300,
+  "max_steps": 8288,
+  "num_train_epochs": 16,
+  "save_steps": 1000,
+  "total_flos": 2.5970934260077363e+17,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3f853383e8dce9f00867dad3f3b4917bf63575060d38b299cdecb6532211b41
+size 4536