we ball

adapter_config.json

@@ -14,12 +14,12 @@
   "r": 32,
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
+    "gate_proj",
     "v_proj",
+    "down_proj",
     "k_proj",
-    "gate_proj",
+    "q_proj",
+    "up_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f56b8a333605f03b496496aac3531e5eb50e390d67be06083619275a78de77da
+oid sha256:8a26259b6c7f10eacd37169a51779a24aa9d6a76d8fdef027422bdcbf2557c2f
 size 500897101

checkpoint-58000/README.md

@@ -1,20 +0,0 @@
----
-library_name: peft
----
-## Training procedure
-
-
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-### Framework versions
-
-
-- PEFT 0.5.0.dev0

checkpoint-58000/adapter_config.json

@@ -1,26 +0,0 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
-  "bias": "none",
-  "fan_in_fan_out": null,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 16,
-  "lora_dropout": 0.05,
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 32,
-  "revision": null,
-  "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "gate_proj",
-    "o_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}

checkpoint-58000/adapter_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf9efdf73d7ecc9f45ca166bec5b70555182c38338e6de139c6203b8a009fc59
-size 500897101

checkpoint-58000/adapter_model/README.md

@@ -1,20 +0,0 @@
----
-library_name: peft
----
-## Training procedure
-
-
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-### Framework versions
-
-
-- PEFT 0.5.0.dev0

checkpoint-58000/adapter_model/adapter_config.json

@@ -1,26 +0,0 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
-  "bias": "none",
-  "fan_in_fan_out": null,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 16,
-  "lora_dropout": 0.05,
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 32,
-  "revision": null,
-  "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "gate_proj",
-    "o_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}

checkpoint-58000/adapter_model/adapter_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cf9efdf73d7ecc9f45ca166bec5b70555182c38338e6de139c6203b8a009fc59
-size 500897101

checkpoint-59000/README.md

@@ -1,20 +0,0 @@
----
-library_name: peft
----
-## Training procedure
-
-
-The following `bitsandbytes` quantization config was used during training:
-- load_in_8bit: False
-- load_in_4bit: True
-- llm_int8_threshold: 6.0
-- llm_int8_skip_modules: None
-- llm_int8_enable_fp32_cpu_offload: False
-- llm_int8_has_fp16_weight: False
-- bnb_4bit_quant_type: nf4
-- bnb_4bit_use_double_quant: True
-- bnb_4bit_compute_dtype: bfloat16
-### Framework versions
-
-
-- PEFT 0.5.0.dev0

checkpoint-59000/adapter_config.json

@@ -1,26 +0,0 @@
-{
-  "auto_mapping": null,
-  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
-  "bias": "none",
-  "fan_in_fan_out": null,
-  "inference_mode": true,
-  "init_lora_weights": true,
-  "layers_pattern": null,
-  "layers_to_transform": null,
-  "lora_alpha": 16,
-  "lora_dropout": 0.05,
-  "modules_to_save": null,
-  "peft_type": "LORA",
-  "r": 32,
-  "revision": null,
-  "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
-    "v_proj",
-    "k_proj",
-    "gate_proj",
-    "o_proj"
-  ],
-  "task_type": "CAUSAL_LM"
-}

checkpoint-59000/adapter_model.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f56b8a333605f03b496496aac3531e5eb50e390d67be06083619275a78de77da
-size 500897101

{checkpoint-56000 → checkpoint-69000}/README.md

@@ -5,6 +5,7 @@ library_name: peft
 
 
 The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
 - load_in_8bit: False
 - load_in_4bit: True
 - llm_int8_threshold: 6.0

{checkpoint-56000/adapter_model → checkpoint-69000}/adapter_config.json

@@ -14,12 +14,12 @@
   "r": 32,
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
+    "gate_proj",
     "v_proj",
+    "down_proj",
     "k_proj",
-    "gate_proj",
+    "q_proj",
+    "up_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

{checkpoint-56000/adapter_model → checkpoint-69000}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d61b2ab661f17f1b28e07a6ea4c559efd2487b69440f512fbda902147b2007f
+oid sha256:16127581d1b65765200af747a5c98d27b237b49430e306dfd23a9c3ad6af3b9c
 size 500897101

{checkpoint-57000 → checkpoint-69000/adapter_model}/README.md

@@ -5,6 +5,7 @@ library_name: peft
 
 
 The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
 - load_in_8bit: False
 - load_in_4bit: True
 - llm_int8_threshold: 6.0

{checkpoint-56000 → checkpoint-69000/adapter_model}/adapter_config.json

@@ -14,12 +14,12 @@
   "r": 32,
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
+    "gate_proj",
     "v_proj",
+    "down_proj",
     "k_proj",
-    "gate_proj",
+    "q_proj",
+    "up_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

{checkpoint-57000 → checkpoint-69000/adapter_model}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9463fbc37a2c37f850b2aa713212bd675cce373b2a226f9fecf647f60157d1a1
+oid sha256:16127581d1b65765200af747a5c98d27b237b49430e306dfd23a9c3ad6af3b9c
 size 500897101

{checkpoint-57000 → checkpoint-69000}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d85e0cce4ea774ece1bba3b083129dd4ea4f075278346655fd271c9663edf7a0
-size 1001723453
+oid sha256:52478f59ec5c65d4db6d79009fc0c477e003ba9db2b5648781779b6963bc40cb
+size 1001724605

{checkpoint-58000 → checkpoint-69000}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5923ba7d43395d2ab7a25af40d67f773d9e67e462f9250548814d4e4d1853054
+oid sha256:b7732edd0ae5999edb700e14bae64e828df5241beb83fbee05815f6c10b73570
 size 14575

{checkpoint-58000 → checkpoint-69000}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb6d23b542a910d4d880a9ad37544effe8607b091db3f3b955d778af0357176f
+oid sha256:f0f5690258b17f07cbd583d2e586e1be27217d957aa1adadeb296ee58f808a87
 size 627

{checkpoint-59000 → checkpoint-69000}/trainer_state.json

@@ -1,8 +1,9 @@
 {
-  "best_metric": 0.4893116354942322,
-  "best_model_checkpoint": "./qlora-out/checkpoint-59000",
-  "epoch": 2.1997688378509377,
-  "global_step": 59000,
+  "best_metric": 0.4789520502090454,
+  "best_model_checkpoint": "./qlora-out/checkpoint-69000",
+  "epoch": 2.5726110137578764,
+  "eval_steps": 500,
+  "global_step": 69000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4018,11 +4019,293 @@
       "eval_samples_per_second": 0.436,
       "eval_steps_per_second": 0.436,
       "step": 59000
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 3.167411635594364e-05,
+      "loss": 0.3867,
+      "step": 59500
+    },
+    {
+      "epoch": 2.22,
+      "eval_loss": 0.48985520005226135,
+      "eval_runtime": 1240.4608,
+      "eval_samples_per_second": 0.437,
+      "eval_steps_per_second": 0.437,
+      "step": 59500
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 3.0261604379828834e-05,
+      "loss": 0.3736,
+      "step": 60000
+    },
+    {
+      "epoch": 2.24,
+      "eval_loss": 0.489548921585083,
+      "eval_runtime": 1234.7527,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 60000
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 2.887567598106955e-05,
+      "loss": 0.361,
+      "step": 60500
+    },
+    {
+      "epoch": 2.26,
+      "eval_loss": 0.4885287582874298,
+      "eval_runtime": 1231.4045,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 60500
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 2.7516859461678857e-05,
+      "loss": 0.3778,
+      "step": 61000
+    },
+    {
+      "epoch": 2.27,
+      "eval_loss": 0.4883672893047333,
+      "eval_runtime": 1235.8497,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 61000
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 2.618567278889328e-05,
+      "loss": 0.3791,
+      "step": 61500
+    },
+    {
+      "epoch": 2.29,
+      "eval_loss": 0.4874744415283203,
+      "eval_runtime": 1231.8195,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 61500
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 2.4882623397728655e-05,
+      "loss": 0.3705,
+      "step": 62000
+    },
+    {
+      "epoch": 2.31,
+      "eval_loss": 0.486933171749115,
+      "eval_runtime": 1227.5583,
+      "eval_samples_per_second": 0.442,
+      "eval_steps_per_second": 0.442,
+      "step": 62000
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 2.3608207997551255e-05,
+      "loss": 0.3698,
+      "step": 62500
+    },
+    {
+      "epoch": 2.33,
+      "eval_loss": 0.48592954874038696,
+      "eval_runtime": 1282.2531,
+      "eval_samples_per_second": 0.423,
+      "eval_steps_per_second": 0.423,
+      "step": 62500
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 2.2362912382736857e-05,
+      "loss": 0.381,
+      "step": 63000
+    },
+    {
+      "epoch": 2.35,
+      "eval_loss": 0.4852922856807709,
+      "eval_runtime": 1229.4457,
+      "eval_samples_per_second": 0.441,
+      "eval_steps_per_second": 0.441,
+      "step": 63000
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 2.1147211247491084e-05,
+      "loss": 0.3728,
+      "step": 63500
+    },
+    {
+      "epoch": 2.37,
+      "eval_loss": 0.484967440366745,
+      "eval_runtime": 1296.2845,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 63500
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.9961568004900565e-05,
+      "loss": 0.3695,
+      "step": 64000
+    },
+    {
+      "epoch": 2.39,
+      "eval_loss": 0.4844016432762146,
+      "eval_runtime": 1317.5418,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 64000
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.8806434610284497e-05,
+      "loss": 0.3682,
+      "step": 64500
+    },
+    {
+      "epoch": 2.4,
+      "eval_loss": 0.4838670790195465,
+      "eval_runtime": 1337.5922,
+      "eval_samples_per_second": 0.405,
+      "eval_steps_per_second": 0.405,
+      "step": 64500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.768225138891393e-05,
+      "loss": 0.3594,
+      "step": 65000
+    },
+    {
+      "epoch": 2.42,
+      "eval_loss": 0.48305046558380127,
+      "eval_runtime": 1317.2888,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 65000
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.6589446868164037e-05,
+      "loss": 0.367,
+      "step": 65500
+    },
+    {
+      "epoch": 2.44,
+      "eval_loss": 0.48225167393684387,
+      "eval_runtime": 1315.9763,
+      "eval_samples_per_second": 0.412,
+      "eval_steps_per_second": 0.412,
+      "step": 65500
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 1.552843761416395e-05,
+      "loss": 0.3781,
+      "step": 66000
+    },
+    {
+      "epoch": 2.46,
+      "eval_loss": 0.48182958364486694,
+      "eval_runtime": 1298.0711,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66000
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 1.4499628073005733e-05,
+      "loss": 0.3632,
+      "step": 66500
+    },
+    {
+      "epoch": 2.48,
+      "eval_loss": 0.48136985301971436,
+      "eval_runtime": 1295.6256,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66500
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 1.350341041657378e-05,
+      "loss": 0.3707,
+      "step": 67000
+    },
+    {
+      "epoch": 2.5,
+      "eval_loss": 0.48081424832344055,
+      "eval_runtime": 1297.8801,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 67000
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 1.2540164393052622e-05,
+      "loss": 0.3657,
+      "step": 67500
+    },
+    {
+      "epoch": 2.52,
+      "eval_loss": 0.48031187057495117,
+      "eval_runtime": 1299.2471,
+      "eval_samples_per_second": 0.417,
+      "eval_steps_per_second": 0.417,
+      "step": 67500
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 1.1610257182170914e-05,
+      "loss": 0.3742,
+      "step": 68000
+    },
+    {
+      "epoch": 2.54,
+      "eval_loss": 0.479922354221344,
+      "eval_runtime": 1275.2567,
+      "eval_samples_per_second": 0.425,
+      "eval_steps_per_second": 0.425,
+      "step": 68000
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 1.0714043255236094e-05,
+      "loss": 0.3761,
+      "step": 68500
+    },
+    {
+      "epoch": 2.55,
+      "eval_loss": 0.4795922338962555,
+      "eval_runtime": 1321.5276,
+      "eval_samples_per_second": 0.41,
+      "eval_steps_per_second": 0.41,
+      "step": 68500
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 9.851864240013509e-06,
+      "loss": 0.3754,
+      "step": 69000
+    },
+    {
+      "epoch": 2.57,
+      "eval_loss": 0.4789520502090454,
+      "eval_runtime": 1345.4528,
+      "eval_samples_per_second": 0.403,
+      "eval_steps_per_second": 0.403,
+      "step": 69000
     }
   ],
+  "logging_steps": 500,
   "max_steps": 80463,
   "num_train_epochs": 3,
-  "total_flos": 1.6542001385066742e+19,
+  "save_steps": 500,
+  "total_flos": 1.9364073941589443e+19,
   "trial_name": null,
   "trial_params": null
 }

{checkpoint-56000 → checkpoint-69000}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ca8c55b410908f1a6fb4d78d55fe6aad82bbca76ec8021e18981496f18fa70
-size 4027
+oid sha256:35750ce2c97e67db338d1121db50269062def2ea29de48747dfd43b7a072ee79
+size 4155

{checkpoint-57000/adapter_model → checkpoint-69500}/README.md

@@ -5,6 +5,7 @@ library_name: peft
 
 
 The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
 - load_in_8bit: False
 - load_in_4bit: True
 - llm_int8_threshold: 6.0

{checkpoint-57000 → checkpoint-69500}/adapter_config.json

@@ -14,12 +14,12 @@
   "r": 32,
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
+    "gate_proj",
     "v_proj",
+    "down_proj",
     "k_proj",
-    "gate_proj",
+    "q_proj",
+    "up_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

{checkpoint-56000 → checkpoint-69500}/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d61b2ab661f17f1b28e07a6ea4c559efd2487b69440f512fbda902147b2007f
+oid sha256:3d536051f2a1ab536e6e716808efa406b8fc4bc641ebcf6102a663de9eab5ffe
 size 500897101

{checkpoint-56000 → checkpoint-69500}/adapter_model/README.md

@@ -5,6 +5,7 @@ library_name: peft
 
 
 The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
 - load_in_8bit: False
 - load_in_4bit: True
 - llm_int8_threshold: 6.0

{checkpoint-57000 → checkpoint-69500}/adapter_model/adapter_config.json

@@ -14,12 +14,12 @@
   "r": 32,
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "down_proj",
-    "q_proj",
+    "gate_proj",
     "v_proj",
+    "down_proj",
     "k_proj",
-    "gate_proj",
+    "q_proj",
+    "up_proj",
     "o_proj"
   ],
   "task_type": "CAUSAL_LM"

{checkpoint-57000 → checkpoint-69500}/adapter_model/adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9463fbc37a2c37f850b2aa713212bd675cce373b2a226f9fecf647f60157d1a1
+oid sha256:3d536051f2a1ab536e6e716808efa406b8fc4bc641ebcf6102a663de9eab5ffe
 size 500897101

{checkpoint-56000 → checkpoint-69500}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0d0299416431a6687f29eb725bd8536e5bc5512ff27981755266d125bd960dc
-size 1001723453
+oid sha256:a0df421a10c3784a131b0ab37e1485ed063b6fa56024cc56104f9dbaad09ebe1
+size 1001724605

{checkpoint-59000 → checkpoint-69500}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bd334de4d3525ea70c0977c8fe7956563ce9e7d3af12dc2b9fcbbc68894cb2d
+oid sha256:c9ee221e71303f97217b0d58a1364dcc9e4c1fac4ba0baf829b9e79b7ae1680b
 size 14575

{checkpoint-56000 → checkpoint-69500}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:40f07a3bf2b8b7e85bd7ec32b459bd8eba34e3ffd70129884ee8cac79708a84f
+oid sha256:bca69d6e74edb4d1fa3e9c45efbdb18d22e7412cb25b7cb947ef97719376c1f2
 size 627

{checkpoint-58000 → checkpoint-69500}/trainer_state.json

@@ -1,8 +1,9 @@
 {
-  "best_metric": 0.4916069805622101,
-  "best_model_checkpoint": "./qlora-out/checkpoint-58000",
-  "epoch": 2.1624846202602437,
-  "global_step": 58000,
+  "best_metric": 0.47866225242614746,
+  "best_model_checkpoint": "./qlora-out/checkpoint-69500",
+  "epoch": 2.591253122553223,
+  "eval_steps": 500,
+  "global_step": 69500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3950,11 +3951,375 @@
       "eval_samples_per_second": 0.436,
       "eval_steps_per_second": 0.436,
       "step": 58000
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.576595345767464e-05,
+      "loss": 0.3759,
+      "step": 58100
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.5467166397551524e-05,
+      "loss": 0.3987,
+      "step": 58200
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.5169363338208094e-05,
+      "loss": 0.3809,
+      "step": 58300
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.4872548820564455e-05,
+      "loss": 0.3851,
+      "step": 58400
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.457672737046737e-05,
+      "loss": 0.3832,
+      "step": 58500
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.42819034986213e-05,
+      "loss": 0.3923,
+      "step": 58600
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 3.398808170051951e-05,
+      "loss": 0.3609,
+      "step": 58700
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 3.369526645637556e-05,
+      "loss": 0.3538,
+      "step": 58800
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 3.3403462231055107e-05,
+      "loss": 0.3941,
+      "step": 58900
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 3.3112673474007584e-05,
+      "loss": 0.3984,
+      "step": 59000
+    },
+    {
+      "epoch": 2.2,
+      "eval_loss": 0.4893116354942322,
+      "eval_runtime": 1243.7748,
+      "eval_samples_per_second": 0.436,
+      "eval_steps_per_second": 0.436,
+      "step": 59000
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 3.167411635594364e-05,
+      "loss": 0.3867,
+      "step": 59500
+    },
+    {
+      "epoch": 2.22,
+      "eval_loss": 0.48985520005226135,
+      "eval_runtime": 1240.4608,
+      "eval_samples_per_second": 0.437,
+      "eval_steps_per_second": 0.437,
+      "step": 59500
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 3.0261604379828834e-05,
+      "loss": 0.3736,
+      "step": 60000
+    },
+    {
+      "epoch": 2.24,
+      "eval_loss": 0.489548921585083,
+      "eval_runtime": 1234.7527,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 60000
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 2.887567598106955e-05,
+      "loss": 0.361,
+      "step": 60500
+    },
+    {
+      "epoch": 2.26,
+      "eval_loss": 0.4885287582874298,
+      "eval_runtime": 1231.4045,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 60500
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 2.7516859461678857e-05,
+      "loss": 0.3778,
+      "step": 61000
+    },
+    {
+      "epoch": 2.27,
+      "eval_loss": 0.4883672893047333,
+      "eval_runtime": 1235.8497,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 61000
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 2.618567278889328e-05,
+      "loss": 0.3791,
+      "step": 61500
+    },
+    {
+      "epoch": 2.29,
+      "eval_loss": 0.4874744415283203,
+      "eval_runtime": 1231.8195,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 61500
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 2.4882623397728655e-05,
+      "loss": 0.3705,
+      "step": 62000
+    },
+    {
+      "epoch": 2.31,
+      "eval_loss": 0.486933171749115,
+      "eval_runtime": 1227.5583,
+      "eval_samples_per_second": 0.442,
+      "eval_steps_per_second": 0.442,
+      "step": 62000
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 2.3608207997551255e-05,
+      "loss": 0.3698,
+      "step": 62500
+    },
+    {
+      "epoch": 2.33,
+      "eval_loss": 0.48592954874038696,
+      "eval_runtime": 1282.2531,
+      "eval_samples_per_second": 0.423,
+      "eval_steps_per_second": 0.423,
+      "step": 62500
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 2.2362912382736857e-05,
+      "loss": 0.381,
+      "step": 63000
+    },
+    {
+      "epoch": 2.35,
+      "eval_loss": 0.4852922856807709,
+      "eval_runtime": 1229.4457,
+      "eval_samples_per_second": 0.441,
+      "eval_steps_per_second": 0.441,
+      "step": 63000
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 2.1147211247491084e-05,
+      "loss": 0.3728,
+      "step": 63500
+    },
+    {
+      "epoch": 2.37,
+      "eval_loss": 0.484967440366745,
+      "eval_runtime": 1296.2845,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 63500
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.9961568004900565e-05,
+      "loss": 0.3695,
+      "step": 64000
+    },
+    {
+      "epoch": 2.39,
+      "eval_loss": 0.4844016432762146,
+      "eval_runtime": 1317.5418,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 64000
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.8806434610284497e-05,
+      "loss": 0.3682,
+      "step": 64500
+    },
+    {
+      "epoch": 2.4,
+      "eval_loss": 0.4838670790195465,
+      "eval_runtime": 1337.5922,
+      "eval_samples_per_second": 0.405,
+      "eval_steps_per_second": 0.405,
+      "step": 64500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.768225138891393e-05,
+      "loss": 0.3594,
+      "step": 65000
+    },
+    {
+      "epoch": 2.42,
+      "eval_loss": 0.48305046558380127,
+      "eval_runtime": 1317.2888,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 65000
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.6589446868164037e-05,
+      "loss": 0.367,
+      "step": 65500
+    },
+    {
+      "epoch": 2.44,
+      "eval_loss": 0.48225167393684387,
+      "eval_runtime": 1315.9763,
+      "eval_samples_per_second": 0.412,
+      "eval_steps_per_second": 0.412,
+      "step": 65500
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 1.552843761416395e-05,
+      "loss": 0.3781,
+      "step": 66000
+    },
+    {
+      "epoch": 2.46,
+      "eval_loss": 0.48182958364486694,
+      "eval_runtime": 1298.0711,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66000
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 1.4499628073005733e-05,
+      "loss": 0.3632,
+      "step": 66500
+    },
+    {
+      "epoch": 2.48,
+      "eval_loss": 0.48136985301971436,
+      "eval_runtime": 1295.6256,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66500
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 1.350341041657378e-05,
+      "loss": 0.3707,
+      "step": 67000
+    },
+    {
+      "epoch": 2.5,
+      "eval_loss": 0.48081424832344055,
+      "eval_runtime": 1297.8801,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 67000
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 1.2540164393052622e-05,
+      "loss": 0.3657,
+      "step": 67500
+    },
+    {
+      "epoch": 2.52,
+      "eval_loss": 0.48031187057495117,
+      "eval_runtime": 1299.2471,
+      "eval_samples_per_second": 0.417,
+      "eval_steps_per_second": 0.417,
+      "step": 67500
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 1.1610257182170914e-05,
+      "loss": 0.3742,
+      "step": 68000
+    },
+    {
+      "epoch": 2.54,
+      "eval_loss": 0.479922354221344,
+      "eval_runtime": 1275.2567,
+      "eval_samples_per_second": 0.425,
+      "eval_steps_per_second": 0.425,
+      "step": 68000
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 1.0714043255236094e-05,
+      "loss": 0.3761,
+      "step": 68500
+    },
+    {
+      "epoch": 2.55,
+      "eval_loss": 0.4795922338962555,
+      "eval_runtime": 1321.5276,
+      "eval_samples_per_second": 0.41,
+      "eval_steps_per_second": 0.41,
+      "step": 68500
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 9.851864240013509e-06,
+      "loss": 0.3754,
+      "step": 69000
+    },
+    {
+      "epoch": 2.57,
+      "eval_loss": 0.4789520502090454,
+      "eval_runtime": 1345.4528,
+      "eval_samples_per_second": 0.403,
+      "eval_steps_per_second": 0.403,
+      "step": 69000
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 9.024048790501272e-06,
+      "loss": 0.3594,
+      "step": 69500
+    },
+    {
+      "epoch": 2.59,
+      "eval_loss": 0.47866225242614746,
+      "eval_runtime": 1316.9883,
+      "eval_samples_per_second": 0.412,
+      "eval_steps_per_second": 0.412,
+      "step": 69500
     }
   ],
+  "logging_steps": 500,
   "max_steps": 80463,
   "num_train_epochs": 3,
-  "total_flos": 1.6261229153876214e+19,
+  "save_steps": 500,
+  "total_flos": 1.950603151563399e+19,
   "trial_name": null,
   "trial_params": null
 }

{checkpoint-58000 → checkpoint-69500}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ca8c55b410908f1a6fb4d78d55fe6aad82bbca76ec8021e18981496f18fa70
-size 4027
+oid sha256:35750ce2c97e67db338d1121db50269062def2ea29de48747dfd43b7a072ee79
+size 4155

checkpoint-70000/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-70000/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-70000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35b27172603bfaa42af020910d0f3a0724656396738e74f39eebef1c4c53cd6c
+size 500897101

checkpoint-70000/adapter_model/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-70000/adapter_model/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-70000/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35b27172603bfaa42af020910d0f3a0724656396738e74f39eebef1c4c53cd6c
+size 500897101

{checkpoint-58000 → checkpoint-70000}/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:67f192e31625a5f9d71aaeb75826e3461458c994c58bc8d3d5b3b59fa56efc4b
-size 1001723453
+oid sha256:3757834dca752ceb36448c74c65b6c698a3cf7eac3b443be1d20520a1ef75c80
+size 1001724605

{checkpoint-57000 → checkpoint-70000}/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7b0ae395ccd0b4875fa94f8cd4ee3274662f44279f744979610604a15d72da0
+oid sha256:e3b4a721a0714cca4311a027981bf55d9c240a69a7f46c912f368eb795c5d17f
 size 14575

{checkpoint-57000 → checkpoint-70000}/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c18b73ff8e0ca9bda5d92134e841aafa154377e996a5dd3b1b1a3a0b329e74e
+oid sha256:d978803312071ed04341fcce57866c271d97c1ced7225c7be19f70453e4d9836
 size 627

{checkpoint-57000 → checkpoint-70000}/trainer_state.json

@@ -1,8 +1,9 @@
 {
-  "best_metric": 0.49361398816108704,
-  "best_model_checkpoint": "./qlora-out/checkpoint-57000",
-  "epoch": 2.12520040266955,
-  "global_step": 57000,
+  "best_metric": 0.47838443517684937,
+  "best_model_checkpoint": "./qlora-out/checkpoint-70000",
+  "epoch": 2.6098952313485704,
+  "eval_steps": 500,
+  "global_step": 70000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3882,11 +3883,457 @@
       "eval_samples_per_second": 0.449,
       "eval_steps_per_second": 0.449,
       "step": 57000
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 3.8806934461220826e-05,
+      "loss": 0.3512,
+      "step": 57100
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 3.8498560410244546e-05,
+      "loss": 0.3715,
+      "step": 57200
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 3.819112413715791e-05,
+      "loss": 0.3803,
+      "step": 57300
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 3.7884630329768875e-05,
+      "loss": 0.3785,
+      "step": 57400
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 3.757908366151463e-05,
+      "loss": 0.3626,
+      "step": 57500
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 3.72744887913904e-05,
+      "loss": 0.3981,
+      "step": 57600
+    },
+    {
+      "epoch": 2.15,
+      "learning_rate": 3.697085036387822e-05,
+      "loss": 0.3918,
+      "step": 57700
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 3.6668173008876324e-05,
+      "loss": 0.3876,
+      "step": 57800
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 3.6366461341628396e-05,
+      "loss": 0.3878,
+      "step": 57900
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 3.606571996265321e-05,
+      "loss": 0.3674,
+      "step": 58000
+    },
+    {
+      "epoch": 2.16,
+      "eval_loss": 0.4916069805622101,
+      "eval_runtime": 1244.109,
+      "eval_samples_per_second": 0.436,
+      "eval_steps_per_second": 0.436,
+      "step": 58000
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.576595345767464e-05,
+      "loss": 0.3759,
+      "step": 58100
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.5467166397551524e-05,
+      "loss": 0.3987,
+      "step": 58200
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 3.5169363338208094e-05,
+      "loss": 0.3809,
+      "step": 58300
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.4872548820564455e-05,
+      "loss": 0.3851,
+      "step": 58400
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.457672737046737e-05,
+      "loss": 0.3832,
+      "step": 58500
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 3.42819034986213e-05,
+      "loss": 0.3923,
+      "step": 58600
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 3.398808170051951e-05,
+      "loss": 0.3609,
+      "step": 58700
+    },
+    {
+      "epoch": 2.19,
+      "learning_rate": 3.369526645637556e-05,
+      "loss": 0.3538,
+      "step": 58800
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 3.3403462231055107e-05,
+      "loss": 0.3941,
+      "step": 58900
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 3.3112673474007584e-05,
+      "loss": 0.3984,
+      "step": 59000
+    },
+    {
+      "epoch": 2.2,
+      "eval_loss": 0.4893116354942322,
+      "eval_runtime": 1243.7748,
+      "eval_samples_per_second": 0.436,
+      "eval_steps_per_second": 0.436,
+      "step": 59000
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 3.167411635594364e-05,
+      "loss": 0.3867,
+      "step": 59500
+    },
+    {
+      "epoch": 2.22,
+      "eval_loss": 0.48985520005226135,
+      "eval_runtime": 1240.4608,
+      "eval_samples_per_second": 0.437,
+      "eval_steps_per_second": 0.437,
+      "step": 59500
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 3.0261604379828834e-05,
+      "loss": 0.3736,
+      "step": 60000
+    },
+    {
+      "epoch": 2.24,
+      "eval_loss": 0.489548921585083,
+      "eval_runtime": 1234.7527,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 60000
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 2.887567598106955e-05,
+      "loss": 0.361,
+      "step": 60500
+    },
+    {
+      "epoch": 2.26,
+      "eval_loss": 0.4885287582874298,
+      "eval_runtime": 1231.4045,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 60500
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 2.7516859461678857e-05,
+      "loss": 0.3778,
+      "step": 61000
+    },
+    {
+      "epoch": 2.27,
+      "eval_loss": 0.4883672893047333,
+      "eval_runtime": 1235.8497,
+      "eval_samples_per_second": 0.439,
+      "eval_steps_per_second": 0.439,
+      "step": 61000
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 2.618567278889328e-05,
+      "loss": 0.3791,
+      "step": 61500
+    },
+    {
+      "epoch": 2.29,
+      "eval_loss": 0.4874744415283203,
+      "eval_runtime": 1231.8195,
+      "eval_samples_per_second": 0.44,
+      "eval_steps_per_second": 0.44,
+      "step": 61500
+    },
+    {
+      "epoch": 2.31,
+      "learning_rate": 2.4882623397728655e-05,
+      "loss": 0.3705,
+      "step": 62000
+    },
+    {
+      "epoch": 2.31,
+      "eval_loss": 0.486933171749115,
+      "eval_runtime": 1227.5583,
+      "eval_samples_per_second": 0.442,
+      "eval_steps_per_second": 0.442,
+      "step": 62000
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 2.3608207997551255e-05,
+      "loss": 0.3698,
+      "step": 62500
+    },
+    {
+      "epoch": 2.33,
+      "eval_loss": 0.48592954874038696,
+      "eval_runtime": 1282.2531,
+      "eval_samples_per_second": 0.423,
+      "eval_steps_per_second": 0.423,
+      "step": 62500
+    },
+    {
+      "epoch": 2.35,
+      "learning_rate": 2.2362912382736857e-05,
+      "loss": 0.381,
+      "step": 63000
+    },
+    {
+      "epoch": 2.35,
+      "eval_loss": 0.4852922856807709,
+      "eval_runtime": 1229.4457,
+      "eval_samples_per_second": 0.441,
+      "eval_steps_per_second": 0.441,
+      "step": 63000
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 2.1147211247491084e-05,
+      "loss": 0.3728,
+      "step": 63500
+    },
+    {
+      "epoch": 2.37,
+      "eval_loss": 0.484967440366745,
+      "eval_runtime": 1296.2845,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 63500
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 1.9961568004900565e-05,
+      "loss": 0.3695,
+      "step": 64000
+    },
+    {
+      "epoch": 2.39,
+      "eval_loss": 0.4844016432762146,
+      "eval_runtime": 1317.5418,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 64000
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.8806434610284497e-05,
+      "loss": 0.3682,
+      "step": 64500
+    },
+    {
+      "epoch": 2.4,
+      "eval_loss": 0.4838670790195465,
+      "eval_runtime": 1337.5922,
+      "eval_samples_per_second": 0.405,
+      "eval_steps_per_second": 0.405,
+      "step": 64500
+    },
+    {
+      "epoch": 2.42,
+      "learning_rate": 1.768225138891393e-05,
+      "loss": 0.3594,
+      "step": 65000
+    },
+    {
+      "epoch": 2.42,
+      "eval_loss": 0.48305046558380127,
+      "eval_runtime": 1317.2888,
+      "eval_samples_per_second": 0.411,
+      "eval_steps_per_second": 0.411,
+      "step": 65000
+    },
+    {
+      "epoch": 2.44,
+      "learning_rate": 1.6589446868164037e-05,
+      "loss": 0.367,
+      "step": 65500
+    },
+    {
+      "epoch": 2.44,
+      "eval_loss": 0.48225167393684387,
+      "eval_runtime": 1315.9763,
+      "eval_samples_per_second": 0.412,
+      "eval_steps_per_second": 0.412,
+      "step": 65500
+    },
+    {
+      "epoch": 2.46,
+      "learning_rate": 1.552843761416395e-05,
+      "loss": 0.3781,
+      "step": 66000
+    },
+    {
+      "epoch": 2.46,
+      "eval_loss": 0.48182958364486694,
+      "eval_runtime": 1298.0711,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66000
+    },
+    {
+      "epoch": 2.48,
+      "learning_rate": 1.4499628073005733e-05,
+      "loss": 0.3632,
+      "step": 66500
+    },
+    {
+      "epoch": 2.48,
+      "eval_loss": 0.48136985301971436,
+      "eval_runtime": 1295.6256,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 66500
+    },
+    {
+      "epoch": 2.5,
+      "learning_rate": 1.350341041657378e-05,
+      "loss": 0.3707,
+      "step": 67000
+    },
+    {
+      "epoch": 2.5,
+      "eval_loss": 0.48081424832344055,
+      "eval_runtime": 1297.8801,
+      "eval_samples_per_second": 0.418,
+      "eval_steps_per_second": 0.418,
+      "step": 67000
+    },
+    {
+      "epoch": 2.52,
+      "learning_rate": 1.2540164393052622e-05,
+      "loss": 0.3657,
+      "step": 67500
+    },
+    {
+      "epoch": 2.52,
+      "eval_loss": 0.48031187057495117,
+      "eval_runtime": 1299.2471,
+      "eval_samples_per_second": 0.417,
+      "eval_steps_per_second": 0.417,
+      "step": 67500
+    },
+    {
+      "epoch": 2.54,
+      "learning_rate": 1.1610257182170914e-05,
+      "loss": 0.3742,
+      "step": 68000
+    },
+    {
+      "epoch": 2.54,
+      "eval_loss": 0.479922354221344,
+      "eval_runtime": 1275.2567,
+      "eval_samples_per_second": 0.425,
+      "eval_steps_per_second": 0.425,
+      "step": 68000
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 1.0714043255236094e-05,
+      "loss": 0.3761,
+      "step": 68500
+    },
+    {
+      "epoch": 2.55,
+      "eval_loss": 0.4795922338962555,
+      "eval_runtime": 1321.5276,
+      "eval_samples_per_second": 0.41,
+      "eval_steps_per_second": 0.41,
+      "step": 68500
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 9.851864240013509e-06,
+      "loss": 0.3754,
+      "step": 69000
+    },
+    {
+      "epoch": 2.57,
+      "eval_loss": 0.4789520502090454,
+      "eval_runtime": 1345.4528,
+      "eval_samples_per_second": 0.403,
+      "eval_steps_per_second": 0.403,
+      "step": 69000
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 9.024048790501272e-06,
+      "loss": 0.3594,
+      "step": 69500
+    },
+    {
+      "epoch": 2.59,
+      "eval_loss": 0.47866225242614746,
+      "eval_runtime": 1316.9883,
+      "eval_samples_per_second": 0.412,
+      "eval_steps_per_second": 0.412,
+      "step": 69500
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 8.230912461650797e-06,
+      "loss": 0.3601,
+      "step": 70000
+    },
+    {
+      "epoch": 2.61,
+      "eval_loss": 0.47838443517684937,
+      "eval_runtime": 1306.7325,
+      "eval_samples_per_second": 0.415,
+      "eval_steps_per_second": 0.415,
+      "step": 70000
     }
   ],
+  "logging_steps": 500,
   "max_steps": 80463,
   "num_train_epochs": 3,
-  "total_flos": 1.5981607298407956e+19,
+  "save_steps": 500,
+  "total_flos": 1.96476655962565e+19,
   "trial_name": null,
   "trial_params": null
 }

{checkpoint-59000 → checkpoint-70000}/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8ca8c55b410908f1a6fb4d78d55fe6aad82bbca76ec8021e18981496f18fa70
-size 4027
+oid sha256:35750ce2c97e67db338d1121db50269062def2ea29de48747dfd43b7a072ee79
+size 4155

checkpoint-70500/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-70500/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-70500/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a26259b6c7f10eacd37169a51779a24aa9d6a76d8fdef027422bdcbf2557c2f
+size 500897101

checkpoint-70500/adapter_model/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+
+- PEFT 0.5.0.dev0

checkpoint-70500/adapter_model/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/workspace/webui/models/TheBloke_Llama-2-13B-fp16",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "revision": null,
+  "target_modules": [
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj",
+    "up_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-70500/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a26259b6c7f10eacd37169a51779a24aa9d6a76d8fdef027422bdcbf2557c2f
+size 500897101