Training in progress, step 2000, checkpoint

checkpoint-2000/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

checkpoint-2000/adapter_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "EleutherAI/polyglot-ko-1.3b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "query_key_value"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-2000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ed689316d9c46cdbf470d9171fc41cba1e4ce89a785742523f44908a1fc538ba
+size 12600958

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98f0404082ad80a7bbc729c38a2265feca96709f29f38136ccd414531a4bae9f
+size 25206586

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79344222b63b458b67e453472a91541e61de32d329369e7abb00c78cdad97f50
+size 14308

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c8b661437e2748974b4081b2121d73c5323798e5067866473bf83006d1ce242
+size 1064

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,72 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|unused0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|unused1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30000": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30001": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30002": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,71 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.644518272425249,
+  "eval_steps": 1000,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2458471760797342e-06,
+      "loss": 2.3433,
+      "step": 300
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 2.4916943521594684e-06,
+      "loss": 2.3014,
+      "step": 600
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 3.737541528239203e-06,
+      "loss": 2.2329,
+      "step": 900
+    },
+    {
+      "epoch": 3.32,
+      "eval_loss": 2.1643757820129395,
+      "eval_runtime": 22.0069,
+      "eval_samples_per_second": 77.112,
+      "eval_steps_per_second": 4.862,
+      "step": 1000
+    },
+    {
+      "epoch": 3.99,
+      "learning_rate": 4.983388704318937e-06,
+      "loss": 2.182,
+      "step": 1200
+    },
+    {
+      "epoch": 4.98,
+      "learning_rate": 6.229235880398672e-06,
+      "loss": 2.1362,
+      "step": 1500
+    },
+    {
+      "epoch": 5.98,
+      "learning_rate": 7.475083056478406e-06,
+      "loss": 2.1095,
+      "step": 1800
+    },
+    {
+      "epoch": 6.64,
+      "eval_loss": 2.0594053268432617,
+      "eval_runtime": 21.9607,
+      "eval_samples_per_second": 77.274,
+      "eval_steps_per_second": 4.872,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 300,
+  "max_steps": 4816,
+  "num_train_epochs": 16,
+  "save_steps": 1000,
+  "total_flos": 1.0446426034200576e+17,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55b0067f61a16c7877c7bed18e4feef50e9a468a1c17ef62ac22d2ec3050fa9e
+size 4536