Training in progress, step 3000, checkpoint

checkpoint-3000/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

checkpoint-3000/adapter_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "EleutherAI/polyglot-ko-1.3b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "query_key_value"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-3000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67b74a984b787aed55852b4bab114533d78e8efd036d80ff03a0bcd1696b49f6
+size 12600958

checkpoint-3000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f3fb213187ee7db6efe381c95f9e552d1b1bc06613f13662aa5c49dd80af69d
+size 25206586

checkpoint-3000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3dbe6d0b236c9fcb9553ba259ac9afb1b0fa947f8c0e08df6551b963eab4b0f0
+size 14308

checkpoint-3000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aad2e439af75803c79b3d25c3ae4d94811fd8e1bafa659fa05ba93ce71c3c21c
+size 1064

checkpoint-3000/special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-3000/tokenizer_config.json

@@ -0,0 +1,72 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|unused0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|unused1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30000": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30001": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30002": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-3000/trainer_state.json

@@ -0,0 +1,103 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.966777408637874,
+  "eval_steps": 1000,
+  "global_step": 3000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2458471760797342e-06,
+      "loss": 2.3433,
+      "step": 300
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 2.4916943521594684e-06,
+      "loss": 2.3014,
+      "step": 600
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 3.737541528239203e-06,
+      "loss": 2.2329,
+      "step": 900
+    },
+    {
+      "epoch": 3.32,
+      "eval_loss": 2.1643757820129395,
+      "eval_runtime": 22.0069,
+      "eval_samples_per_second": 77.112,
+      "eval_steps_per_second": 4.862,
+      "step": 1000
+    },
+    {
+      "epoch": 3.99,
+      "learning_rate": 4.983388704318937e-06,
+      "loss": 2.182,
+      "step": 1200
+    },
+    {
+      "epoch": 4.98,
+      "learning_rate": 6.229235880398672e-06,
+      "loss": 2.1362,
+      "step": 1500
+    },
+    {
+      "epoch": 5.98,
+      "learning_rate": 7.475083056478406e-06,
+      "loss": 2.1095,
+      "step": 1800
+    },
+    {
+      "epoch": 6.64,
+      "eval_loss": 2.0594053268432617,
+      "eval_runtime": 21.9607,
+      "eval_samples_per_second": 77.274,
+      "eval_steps_per_second": 4.872,
+      "step": 2000
+    },
+    {
+      "epoch": 6.98,
+      "learning_rate": 8.72093023255814e-06,
+      "loss": 2.0928,
+      "step": 2100
+    },
+    {
+      "epoch": 7.97,
+      "learning_rate": 9.966777408637874e-06,
+      "loss": 2.0716,
+      "step": 2400
+    },
+    {
+      "epoch": 8.97,
+      "learning_rate": 9.641545735087877e-06,
+      "loss": 2.0567,
+      "step": 2700
+    },
+    {
+      "epoch": 9.97,
+      "learning_rate": 8.581357985235595e-06,
+      "loss": 2.0413,
+      "step": 3000
+    },
+    {
+      "epoch": 9.97,
+      "eval_loss": 2.0132548809051514,
+      "eval_runtime": 21.984,
+      "eval_samples_per_second": 77.192,
+      "eval_steps_per_second": 4.867,
+      "step": 3000
+    }
+  ],
+  "logging_steps": 300,
+  "max_steps": 4816,
+  "num_train_epochs": 16,
+  "save_steps": 1000,
+  "total_flos": 1.5662901235512115e+17,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55b0067f61a16c7877c7bed18e4feef50e9a468a1c17ef62ac22d2ec3050fa9e
+size 4536