Upload 36 files

quality_chunk/all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.37379822503952753,
+    "train_runtime": 12484.1415,
+    "train_samples_per_second": 0.404,
+    "train_steps_per_second": 0.034
+}

quality_chunk/final/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+### Framework versions
+
+
+- PEFT 0.5.0

quality_chunk/final/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data0/maqi/huggingface_models/option1-models/race_ft",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "gate_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

quality_chunk/final/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d78b1a0794ca174244a339da2cb122d8d6685d2f7b55a1e195a7c79b05ea3649
+size 1279424269

quality_chunk/final/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8f6e33414e7f3060d835a12e5b100b720ce1777a24c2d0deca45f67564d1b57
+size 4091

quality_chunk/quality_chunk.json

@@ -0,0 +1,31 @@
+{
+    "desc": "QuALITY 使用随机选择的关键句训练",
+    "output_dir": "output/ablation_study/QuALITY/quality_chunk",
+    "model_name_or_path": "/data0/maqi/huggingface_models/option1-models/race_ft",
+    "train_file": "/data0/maqi/KGLQA-data/datasets/QuALITY/random_select/quality_chunk_2048_instruct/train.jsonl",
+    "num_train_epochs": 2,
+    "per_device_train_batch_size": 6,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 1e-4,
+    "max_seq_length": 2048,
+    "logging_steps": 50,
+    "save_steps": 200,
+    "save_total_limit": 1,
+    "lr_scheduler_type": "constant_with_warmup",
+    "warmup_ratio": 0.1,
+    "lora_rank": 128,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+
+    "gradient_checkpointing": true,
+    "disable_tqdm": false,
+    "optim": "paged_adamw_32bit",
+    "seed": 318,
+    "fp16": true,
+    "report_to": "tensorboard",
+    "dataloader_num_workers": 10,
+    "save_strategy": "steps",
+    "weight_decay": 0,
+    "max_grad_norm": 0.3,
+    "remove_unused_columns": false
+}

quality_chunk/train.log

@@ -0,0 +1,32 @@
+model training desc:  QuALITY 使用随机选择的关键句训练
+2023-12-05 18:56:31.460 | INFO     | __main__:init_components:108 - Initializing components...
+
+You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+2023-12-05 18:57:02.985 | INFO     | __main__:init_components:155 - 
+
+2023-12-05 18:57:02.985 | INFO     | __main__:init_components:156 - ********************
+2023-12-05 18:57:02.985 | INFO     | __main__:init_components:157 - using llama2 model
+2023-12-05 18:57:02.985 | INFO     | __main__:init_components:158 - ********************
+2023-12-05 18:57:02.985 | INFO     | __main__:init_components:159 - 
+
+memory footprint of model: 4.024436950683594 GB
+trainable params: 319,815,680 || all params: 7,058,231,296 || trainable%: 4.531102291607305
+2023-12-05 18:57:44.936 | INFO     | component.dataset:__init__:14 - Loading data: /data0/maqi/KGLQA-data/datasets/QuALITY/random_select/quality_chunk_2048_instruct/train.jsonl
+2023-12-05 18:57:45.028 | INFO     | component.dataset:__init__:19 - there are 2523 data in dataset
+2023-12-05 18:57:45.086 | INFO     | __main__:main:231 - *** starting training ***
+
  0%|          | 0/420 [00:00<?, ?it/s]
  0%|          | 1/420 [00:33<3:55:39, 33.74s/it]
  0%|          | 2/420 [01:03<3:37:45, 31.26s/it]
  1%|          | 3/420 [01:32<3:32:06, 30.52s/it]
  1%|          | 4/420 [02:02<3:28:14, 30.03s/it]
  1%|          | 5/420 [02:32<3:27:39, 30.02s/it]
  1%|▏         | 6/420 [03:02<3:27:09, 30.02s/it]
  2%|▏         | 7/420 [03:31<3:25:11, 29.81s/it]
  2%|▏         | 8/420 [04:00<3:23:43, 29.67s/it]
  2%|▏         | 9/420 [04:30<3:23:20, 29.68s/it]
  2%|▏         | 10/420 [05:00<3:22:54, 29.69s/it]
  3%|▎         | 11/420 [05:30<3:22:29, 29.71s/it]
  3%|▎         | 12/420 [05:59<3:21:19, 29.61s/it]
  3%|▎         | 13/420 [06:29<3:21:03, 29.64s/it]
  3%|▎         | 14/420 [06:58<3:20:44, 29.67s/it]
  4%|▎         | 15/420 [07:28<3:20:20, 29.68s/it]
  4%|▍         | 16/420 [07:58<3:19:14, 29.59s/it]
  4%|▍         | 17/420 [08:27<3:19:00, 29.63s/it]
  4%|▍         | 18/420 [08:57<3:19:20, 29.75s/it]
  5%|▍         | 19/420 [09:27<3:18:46, 29.74s/it]
  5%|▍         | 20/420 [09:57<3:18:54, 29.84s/it]
  5%|▌         | 21/420 [10:27<3:18:11, 29.80s/it]
  5%|▌         | 22/420 [10:56<3:16:51, 29.68s/it]
  5%|▌         | 23/420 [11:26<3:16:26, 29.69s/it]
  6%|▌         | 24/420 [11:56<3:15:57, 29.69s/it]
  6%|▌         | 25/420 [12:25<3:15:30, 29.70s/it]
  6%|▌         | 26/420 [12:55<3:15:01, 29.70s/it]
  6%|▋         | 27/420 [13:25<3:15:13, 29.81s/it]
  7%|▋         | 28/420 [13:55<3:15:14, 29.89s/it]
  7%|▋         | 29/420 [14:25<3:14:20, 29.82s/it]
  7%|▋         | 30/420 [14:55<3:13:35, 29.78s/it]
  7%|▋         | 31/420 [15:24<3:12:16, 29.66s/it]
  8%|▊         | 32/420 [15:54<3:12:29, 29.77s/it]
  8%|▊         | 33/420 [16:23<3:11:12, 29.65s/it]
  8%|▊         | 34/420 [16:53<3:11:30, 29.77s/it]
  8%|▊         | 35/420 [17:23<3:10:53, 29.75s/it]
  9%|▊         | 36/420 [17:53<3:10:57, 29.84s/it]
  9%|▉         | 37/420 [18:23<3:10:53, 29.91s/it]
  9%|▉         | 38/420 [18:53<3:09:59, 29.84s/it]
  9%|▉         | 39/420 [19:23<3:09:53, 29.90s/it]
 10%|▉         | 40/420 [19:52<3:08:25, 29.75s/it]
 10%|▉         | 41/420 [20:22<3:07:13, 29.64s/it]
 10%|█         | 42/420 [20:51<3:06:51, 29.66s/it]
 10%|█         | 43/420 [21:21<3:06:26, 29.67s/it]
 10%|█         | 44/420 [21:51<3:06:40, 29.79s/it]
 11%|█         | 45/420 [22:21<3:06:02, 29.77s/it]
 11%|█         | 46/420 [22:51<3:05:26, 29.75s/it]
 11%|█         | 47/420 [23:20<3:04:15, 29.64s/it]
 11%|█▏        | 48/420 [23:50<3:04:31, 29.76s/it]
 12%|█▏        | 49/420 [24:20<3:03:54, 29.74s/it]
 12%|█▏        | 50/420 [24:49<3:03:20, 29.73s/it]
                                                  
+
 12%|█▏        | 50/420 [24:49<3:03:20, 29.73s/it]
 12%|█▏        | 51/420 [25:19<3:03:31, 29.84s/it]
 12%|█▏        | 52/420 [25:49<3:02:09, 29.70s/it]
 13%|█▎        | 53/420 [26:18<3:01:03, 29.60s/it]
 13%|█▎        | 54/420 [26:48<3:00:46, 29.64s/it]
 13%|█▎        | 55/420 [27:18<3:00:25, 29.66s/it]
 13%|█▎        | 56/420 [27:47<3:00:02, 29.68s/it]
 14%|█▎        | 57/420 [28:17<3:00:13, 29.79s/it]
 14%|█▍        | 58/420 [28:47<2:59:36, 29.77s/it]
 14%|█▍        | 59/420 [29:17<2:59:34, 29.85s/it]
 14%|█▍        | 60/420 [29:47<2:58:50, 29.81s/it]
 15%|█▍        | 61/420 [30:16<2:57:34, 29.68s/it]
 15%|█▍        | 62/420 [30:46<2:57:41, 29.78s/it]
 15%|█▌        | 63/420 [31:16<2:56:19, 29.64s/it]
 15%|█▌        | 64/420 [31:45<2:55:23, 29.56s/it]
 15%|█▌        | 65/420 [32:15<2:55:12, 29.61s/it]
 16%|█▌        | 66/420 [32:44<2:54:17, 29.54s/it]
 16%|█▌        | 67/420 [33:14<2:54:04, 29.59s/it]
 16%|█▌        | 68/420 [33:44<2:54:20, 29.72s/it]
 16%|█▋        | 69/420 [34:14<2:54:23, 29.81s/it]
 17%|█▋        | 70/420 [34:44<2:54:17, 29.88s/it]
 17%|█▋        | 71/420 [35:14<2:53:30, 29.83s/it]
 17%|█▋        | 72/420 [35:43<2:52:09, 29.68s/it]
 17%|█▋        | 73/420 [36:13<2:52:14, 29.78s/it]
 18%|█▊        | 74/420 [36:43<2:52:10, 29.86s/it]
 18%|█▊        | 75/420 [37:13<2:51:25, 29.81s/it]
 18%|█▊        | 76/420 [37:42<2:50:09, 29.68s/it]
 18%|█▊        | 77/420 [38:11<2:49:08, 29.59s/it]
 19%|█▊        | 78/420 [38:41<2:49:21, 29.71s/it]
 19%|█▉        | 79/420 [39:11<2:49:24, 29.81s/it]
 19%|█▉        | 80/420 [39:41<2:49:15, 29.87s/it]
 19%|█▉        | 81/420 [40:11<2:49:01, 29.92s/it]
 20%|█▉        | 82/420 [40:41<2:47:59, 29.82s/it]
 20%|█▉        | 83/420 [41:10<2:46:43, 29.68s/it]
 20%|██        | 84/420 [41:40<2:46:16, 29.69s/it]
 20%|██        | 85/420 [42:10<2:45:47, 29.70s/it]
 20%|██        | 86/420 [42:40<2:45:18, 29.70s/it]
 21%|██        | 87/420 [43:09<2:44:49, 29.70s/it]
 21%|██        | 88/420 [43:39<2:44:19, 29.70s/it]
 21%|██        | 89/420 [44:09<2:44:21, 29.79s/it]
 21%|██▏       | 90/420 [44:38<2:43:07, 29.66s/it]
 22%|██▏       | 91/420 [45:08<2:42:08, 29.57s/it]
 22%|██▏       | 92/420 [45:37<2:41:52, 29.61s/it]
 22%|██▏       | 93/420 [46:07<2:42:03, 29.73s/it]
 22%|██▏       | 94/420 [46:37<2:42:02, 29.82s/it]
 23%|██▎       | 95/420 [47:07<2:40:40, 29.66s/it]
 23%|██▎       | 96/420 [47:37<2:40:46, 29.77s/it]
 23%|██▎       | 97/420 [48:06<2:40:12, 29.76s/it]
 23%|██▎       | 98/420 [48:36<2:40:07, 29.84s/it]
 24%|██▎       | 99/420 [49:06<2:38:42, 29.67s/it]
 24%|██▍       | 100/420 [49:35<2:38:18, 29.68s/it]
                                                   
+
 24%|██▍       | 100/420 [49:35<2:38:18, 29.68s/it]
 24%|██▍       | 101/420 [50:06<2:38:24, 29.80s/it]
 24%|██▍       | 102/420 [50:35<2:37:36, 29.74s/it]
 25%|██▍       | 103/420 [51:05<2:37:06, 29.74s/it]
 25%|██▍       | 104/420 [51:35<2:36:34, 29.73s/it]
 25%|██▌       | 105/420 [52:05<2:36:32, 29.82s/it]
 25%|██▌       | 106/420 [52:35<2:36:24, 29.89s/it]
 25%|██▌       | 107/420 [53:04<2:35:37, 29.83s/it]
 26%|██▌       | 108/420 [53:34<2:35:27, 29.90s/it]
 26%|██▌       | 109/420 [54:04<2:34:28, 29.80s/it]
 26%|██▌       | 110/420 [54:33<2:33:16, 29.66s/it]
 26%|██▋       | 111/420 [55:03<2:32:50, 29.68s/it]
 27%|██▋       | 112/420 [55:33<2:32:55, 29.79s/it]
 27%|██▋       | 113/420 [56:02<2:31:47, 29.67s/it]
 27%|██▋       | 114/420 [56:32<2:30:51, 29.58s/it]
 27%|██▋       | 115/420 [57:02<2:30:33, 29.62s/it]
 28%|██▊       | 116/420 [57:31<2:29:40, 29.54s/it]
 28%|██▊       | 117/420 [58:01<2:29:27, 29.60s/it]
 28%|██▊       | 118/420 [58:31<2:29:35, 29.72s/it]
 28%|██▊       | 119/420 [59:00<2:29:02, 29.71s/it]
 29%|██▊       | 120/420 [59:30<2:28:01, 29.61s/it]
 29%|██▉       | 121/420 [1:00:00<2:28:08, 29.73s/it]
 29%|██▉       | 122/420 [1:00:29<2:27:37, 29.72s/it]
 29%|██▉       | 123/420 [1:00:59<2:27:06, 29.72s/it]
 30%|██▉       | 124/420 [1:01:29<2:26:35, 29.71s/it]
 30%|██▉       | 125/420 [1:01:59<2:26:06, 29.72s/it]
 30%|███       | 126/420 [1:02:28<2:25:38, 29.72s/it]
 30%|███       | 127/420 [1:02:58<2:24:38, 29.62s/it]
 30%|███       | 128/420 [1:03:27<2:23:46, 29.54s/it]
 31%|███       | 129/420 [1:03:57<2:23:58, 29.69s/it]
 31%|███       | 130/420 [1:04:26<2:23:00, 29.59s/it]
 31%|███       | 131/420 [1:04:56<2:22:43, 29.63s/it]
 31%|███▏      | 132/420 [1:05:26<2:22:19, 29.65s/it]
 32%|███▏      | 133/420 [1:05:56<2:21:54, 29.67s/it]
 32%|███▏      | 134/420 [1:06:26<2:21:54, 29.77s/it]
 32%|███▏      | 135/420 [1:06:56<2:21:47, 29.85s/it]
 32%|███▏      | 136/420 [1:07:26<2:21:32, 29.90s/it]
 33%|███▎      | 137/420 [1:07:55<2:20:45, 29.84s/it]
 33%|███▎      | 138/420 [1:08:25<2:20:03, 29.80s/it]
 33%|███▎      | 139/420 [1:08:55<2:19:22, 29.76s/it]
 33%|███▎      | 140/420 [1:09:24<2:18:47, 29.74s/it]
 34%|███▎      | 141/420 [1:09:54<2:18:14, 29.73s/it]
 34%|███▍      | 142/420 [1:10:24<2:18:07, 29.81s/it]
 34%|███▍      | 143/420 [1:10:54<2:17:54, 29.87s/it]
 34%|███▍      | 144/420 [1:11:24<2:17:11, 29.82s/it]
 35%|███▍      | 145/420 [1:11:54<2:16:58, 29.88s/it]
 35%|███▍      | 146/420 [1:12:24<2:16:15, 29.84s/it]
 35%|███▌      | 147/420 [1:12:53<2:15:33, 29.79s/it]
 35%|███▌      | 148/420 [1:13:23<2:14:54, 29.76s/it]
 35%|███▌      | 149/420 [1:13:53<2:14:19, 29.74s/it]
 36%|███▌      | 150/420 [1:14:22<2:13:18, 29.63s/it]
                                                     
+
 36%|███▌      | 150/420 [1:14:22<2:13:18, 29.63s/it]
 36%|███▌      | 151/420 [1:14:52<2:13:20, 29.74s/it]
 36%|███▌      | 152/420 [1:15:21<2:12:19, 29.62s/it]
 36%|███▋      | 153/420 [1:15:51<2:11:55, 29.65s/it]
 37%|███▋      | 154/420 [1:16:21<2:11:56, 29.76s/it]
 37%|███▋      | 155/420 [1:16:50<2:10:54, 29.64s/it]
 37%|███▋      | 156/420 [1:17:20<2:10:31, 29.66s/it]
 37%|███▋      | 157/420 [1:17:50<2:10:31, 29.78s/it]
 38%|███▊      | 158/420 [1:18:20<2:09:56, 29.76s/it]
 38%|███▊      | 159/420 [1:18:50<2:09:46, 29.84s/it]
 38%|███▊      | 160/420 [1:19:20<2:09:07, 29.80s/it]
 38%|███▊      | 161/420 [1:19:49<2:08:32, 29.78s/it]
 39%|███▊      | 162/420 [1:20:19<2:08:21, 29.85s/it]
 39%|███▉      | 163/420 [1:20:49<2:08:03, 29.90s/it]
 39%|███▉      | 164/420 [1:21:19<2:07:19, 29.84s/it]
 39%|███▉      | 165/420 [1:21:49<2:07:04, 29.90s/it]
 40%|███▉      | 166/420 [1:22:19<2:06:21, 29.85s/it]
 40%|███▉      | 167/420 [1:22:49<2:05:43, 29.82s/it]
 40%|████      | 168/420 [1:23:18<2:05:04, 29.78s/it]
 40%|████      | 169/420 [1:23:48<2:04:30, 29.76s/it]
 40%|████      | 170/420 [1:24:18<2:03:57, 29.75s/it]
 41%|████      | 171/420 [1:24:47<2:03:23, 29.73s/it]
 41%|████      | 172/420 [1:25:17<2:02:27, 29.63s/it]
 41%|████      | 173/420 [1:25:47<2:02:29, 29.76s/it]
 41%|████▏     | 174/420 [1:26:16<2:01:48, 29.71s/it]
 42%|████▏     | 175/420 [1:26:46<2:01:19, 29.71s/it]
 42%|████▏     | 176/420 [1:27:16<2:00:25, 29.61s/it]
 42%|████▏     | 177/420 [1:27:45<2:00:02, 29.64s/it]
 42%|████▏     | 178/420 [1:28:15<1:59:37, 29.66s/it]
 43%|████▎     | 179/420 [1:28:45<1:59:34, 29.77s/it]
 43%|████▎     | 180/420 [1:29:15<1:59:00, 29.75s/it]
 43%|████▎     | 181/420 [1:29:44<1:58:27, 29.74s/it]
 43%|████▎     | 182/420 [1:30:14<1:57:31, 29.63s/it]
 44%|████▎     | 183/420 [1:30:44<1:57:30, 29.75s/it]
 44%|████▍     | 184/420 [1:31:14<1:56:57, 29.74s/it]
 44%|████▍     | 185/420 [1:31:43<1:56:25, 29.73s/it]
 44%|████▍     | 186/420 [1:32:13<1:55:29, 29.61s/it]
 45%|████▍     | 187/420 [1:32:42<1:55:06, 29.64s/it]
 45%|████▍     | 188/420 [1:33:12<1:55:06, 29.77s/it]
 45%|████▌     | 189/420 [1:33:42<1:54:54, 29.85s/it]
 45%|████▌     | 190/420 [1:34:12<1:54:15, 29.81s/it]
 45%|████▌     | 191/420 [1:34:42<1:54:00, 29.87s/it]
 46%|████▌     | 192/420 [1:35:12<1:52:56, 29.72s/it]
 46%|████▌     | 193/420 [1:35:41<1:52:26, 29.72s/it]
 46%|████▌     | 194/420 [1:36:11<1:51:32, 29.61s/it]
 46%|████▋     | 195/420 [1:36:40<1:50:45, 29.54s/it]
 47%|████▋     | 196/420 [1:37:10<1:50:28, 29.59s/it]
 47%|████▋     | 197/420 [1:37:40<1:50:28, 29.72s/it]
 47%|████▋     | 198/420 [1:38:10<1:50:16, 29.80s/it]
 47%|████▋     | 199/420 [1:38:40<1:49:59, 29.86s/it]
 48%|████▊     | 200/420 [1:39:09<1:48:50, 29.69s/it]
                                                     
+
 48%|████▊     | 200/420 [1:39:09<1:48:50, 29.69s/it]
 48%|████▊     | 201/420 [1:39:42<1:52:28, 30.81s/it]
 48%|████▊     | 202/420 [1:40:12<1:50:45, 30.49s/it]
 48%|████▊     | 203/420 [1:40:42<1:49:23, 30.25s/it]
 49%|████▊     | 204/420 [1:41:12<1:48:37, 30.17s/it]
 49%|████▉     | 205/420 [1:41:42<1:47:37, 30.03s/it]
 49%|████▉     | 206/420 [1:42:12<1:47:06, 30.03s/it]
 49%|████▉     | 207/420 [1:42:41<1:46:15, 29.93s/it]
 50%|████▉     | 208/420 [1:43:11<1:45:32, 29.87s/it]
 50%|████▉     | 209/420 [1:43:41<1:44:52, 29.82s/it]
 50%|█████     | 210/420 [1:44:10<1:43:52, 29.68s/it]
 50%|█████     | 211/420 [1:44:35<1:38:22, 28.24s/it]
 50%|█████     | 212/420 [1:45:05<1:39:23, 28.67s/it]
 51%|█████     | 213/420 [1:45:35<1:40:18, 29.07s/it]
 51%|█████     | 214/420 [1:46:04<1:40:06, 29.16s/it]
 51%|█████     | 215/420 [1:46:34<1:40:29, 29.41s/it]
 51%|█████▏    | 216/420 [1:47:04<1:40:17, 29.50s/it]
 52%|█████▏    | 217/420 [1:47:34<1:40:21, 29.66s/it]
 52%|█████▏    | 218/420 [1:48:03<1:39:28, 29.55s/it]
 52%|█████▏    | 219/420 [1:48:32<1:38:45, 29.48s/it]
 52%|█████▏    | 220/420 [1:49:02<1:38:30, 29.55s/it]
 53%|█████▎    | 221/420 [1:49:32<1:38:09, 29.59s/it]
 53%|█████▎    | 222/420 [1:50:01<1:37:45, 29.63s/it]
 53%|█████▎    | 223/420 [1:50:31<1:37:20, 29.65s/it]
 53%|█████▎    | 224/420 [1:51:01<1:36:51, 29.65s/it]
 54%|█████▎    | 225/420 [1:51:31<1:36:41, 29.75s/it]
 54%|█████▍    | 226/420 [1:52:01<1:36:26, 29.83s/it]
 54%|█████▍    | 227/420 [1:52:30<1:35:47, 29.78s/it]
 54%|█████▍    | 228/420 [1:53:00<1:35:31, 29.85s/it]
 55%|█████▍    | 229/420 [1:53:30<1:34:52, 29.80s/it]
 55%|█████▍    | 230/420 [1:54:00<1:34:15, 29.77s/it]
 55%|█████▌    | 231/420 [1:54:30<1:33:59, 29.84s/it]
 55%|█████▌    | 232/420 [1:55:00<1:33:20, 29.79s/it]
 55%|█████▌    | 233/420 [1:55:29<1:32:44, 29.76s/it]
 56%|█████▌    | 234/420 [1:55:58<1:31:45, 29.60s/it]
 56%|█████▌    | 235/420 [1:56:28<1:31:20, 29.63s/it]
 56%|█████▌    | 236/420 [1:56:57<1:30:34, 29.54s/it]
 56%|█████▋    | 237/420 [1:57:27<1:30:12, 29.58s/it]
 57%|█████▋    | 238/420 [1:57:57<1:29:50, 29.62s/it]
 57%|█████▋    | 239/420 [1:58:27<1:29:24, 29.64s/it]
 57%|█████▋    | 240/420 [1:58:56<1:28:58, 29.66s/it]
 57%|█████▋    | 241/420 [1:59:26<1:28:30, 29.67s/it]
 58%|█████▊    | 242/420 [1:59:56<1:28:19, 29.77s/it]
 58%|█████▊    | 243/420 [2:00:26<1:27:47, 29.76s/it]
 58%|█████▊    | 244/420 [2:00:56<1:27:31, 29.84s/it]
 58%|█████▊    | 245/420 [2:01:26<1:27:10, 29.89s/it]
 59%|█████▊    | 246/420 [2:01:56<1:26:48, 29.93s/it]
 59%|█████▉    | 247/420 [2:02:26<1:26:21, 29.95s/it]
 59%|█████▉    | 248/420 [2:02:55<1:25:38, 29.87s/it]
 59%|█████▉    | 249/420 [2:03:25<1:24:41, 29.71s/it]
 60%|█████▉    | 250/420 [2:03:54<1:24:10, 29.71s/it]
                                                     
+
 60%|█████▉    | 250/420 [2:03:54<1:24:10, 29.71s/it]
 60%|█████▉    | 251/420 [2:04:24<1:23:41, 29.71s/it]
 60%|██████    | 252/420 [2:04:54<1:23:10, 29.70s/it]
 60%|██████    | 253/420 [2:05:24<1:22:40, 29.70s/it]
 60%|██████    | 254/420 [2:05:53<1:22:10, 29.70s/it]
 61%|██████    | 255/420 [2:06:23<1:21:40, 29.70s/it]
 61%|██████    | 256/420 [2:06:53<1:21:09, 29.69s/it]
 61%|██████    | 257/420 [2:07:22<1:20:39, 29.69s/it]
 61%|██████▏   | 258/420 [2:07:52<1:19:53, 29.59s/it]
 62%|██████▏   | 259/420 [2:08:21<1:19:29, 29.63s/it]
 62%|██████▏   | 260/420 [2:08:51<1:19:18, 29.74s/it]
 62%|██████▏   | 261/420 [2:09:21<1:18:45, 29.72s/it]
 62%|██████▏   | 262/420 [2:09:50<1:17:58, 29.61s/it]
 63%|██████▎   | 263/420 [2:10:20<1:17:32, 29.63s/it]
 63%|██████▎   | 264/420 [2:10:49<1:16:49, 29.55s/it]
 63%|██████▎   | 265/420 [2:11:19<1:16:39, 29.68s/it]
 63%|██████▎   | 266/420 [2:11:49<1:16:24, 29.77s/it]
 64%|██████▎   | 267/420 [2:12:19<1:15:51, 29.75s/it]
 64%|██████▍   | 268/420 [2:12:49<1:15:19, 29.73s/it]
 64%|██████▍   | 269/420 [2:13:18<1:14:47, 29.72s/it]
 64%|██████▍   | 270/420 [2:13:48<1:14:29, 29.79s/it]
 65%|██████▍   | 271/420 [2:14:18<1:13:53, 29.75s/it]
 65%|██████▍   | 272/420 [2:14:48<1:13:21, 29.74s/it]
 65%|██████▌   | 273/420 [2:15:18<1:13:03, 29.82s/it]
 65%|██████▌   | 274/420 [2:15:48<1:12:42, 29.88s/it]
 65%|██████▌   | 275/420 [2:16:17<1:11:49, 29.72s/it]
 66%|██████▌   | 276/420 [2:16:47<1:11:29, 29.79s/it]
 66%|██████▌   | 277/420 [2:17:17<1:10:54, 29.75s/it]
 66%|██████▌   | 278/420 [2:17:46<1:10:07, 29.63s/it]
 66%|██████▋   | 279/420 [2:18:16<1:09:38, 29.64s/it]
 67%|██████▋   | 280/420 [2:18:46<1:09:11, 29.66s/it]
 67%|██████▋   | 281/420 [2:19:16<1:08:56, 29.76s/it]
 67%|██████▋   | 282/420 [2:19:45<1:08:22, 29.73s/it]
 67%|██████▋   | 283/420 [2:20:15<1:07:50, 29.71s/it]
 68%|██████▊   | 284/420 [2:20:45<1:07:32, 29.80s/it]
 68%|██████▊   | 285/420 [2:21:14<1:06:44, 29.66s/it]
 68%|██████▊   | 286/420 [2:21:44<1:06:29, 29.77s/it]
 68%|██████▊   | 287/420 [2:22:14<1:05:55, 29.74s/it]
 69%|██████▊   | 288/420 [2:22:44<1:05:24, 29.73s/it]
 69%|██████▉   | 289/420 [2:23:14<1:05:05, 29.81s/it]
 69%|██████▉   | 290/420 [2:23:43<1:04:17, 29.67s/it]
 69%|██████▉   | 291/420 [2:24:12<1:03:33, 29.57s/it]
 70%|██████▉   | 292/420 [2:24:42<1:03:10, 29.61s/it]
 70%|██████▉   | 293/420 [2:25:12<1:02:38, 29.60s/it]
 70%|███████   | 294/420 [2:25:41<1:02:13, 29.63s/it]
 70%|███████   | 295/420 [2:26:11<1:01:57, 29.74s/it]
 70%|███████   | 296/420 [2:26:41<1:01:13, 29.63s/it]
 71%|███████   | 297/420 [2:27:10<1:00:34, 29.55s/it]
 71%|███████   | 298/420 [2:27:40<1:00:09, 29.59s/it]
 71%|███████   | 299/420 [2:28:09<59:42, 29.61s/it]  
 71%|███████▏  | 300/420 [2:28:39<59:16, 29.64s/it]
                                                   
+
 71%|███████▏  | 300/420 [2:28:39<59:16, 29.64s/it]
 72%|███████▏  | 301/420 [2:29:09<59:00, 29.75s/it]
 72%|███████▏  | 302/420 [2:29:39<58:28, 29.73s/it]
 72%|███████▏  | 303/420 [2:30:08<57:45, 29.62s/it]
 72%|███████▏  | 304/420 [2:30:38<57:18, 29.64s/it]
 73%|███████▎  | 305/420 [2:31:07<56:39, 29.56s/it]
 73%|███████▎  | 306/420 [2:31:36<56:02, 29.50s/it]
 73%|███████▎  | 307/420 [2:32:06<55:49, 29.64s/it]
 73%|███████▎  | 308/420 [2:32:36<55:21, 29.65s/it]
 74%|███████▎  | 309/420 [2:33:06<54:52, 29.66s/it]
 74%|███████▍  | 310/420 [2:33:35<54:22, 29.66s/it]
 74%|███████▍  | 311/420 [2:34:05<54:04, 29.76s/it]
 74%|███████▍  | 312/420 [2:34:35<53:41, 29.83s/it]
 75%|███████▍  | 313/420 [2:35:05<52:56, 29.68s/it]
 75%|███████▍  | 314/420 [2:35:35<52:26, 29.69s/it]
 75%|███████▌  | 315/420 [2:36:04<51:57, 29.69s/it]
 75%|███████▌  | 316/420 [2:36:34<51:26, 29.68s/it]
 75%|███████▌  | 317/420 [2:37:03<50:45, 29.57s/it]
 76%|███████▌  | 318/420 [2:37:33<50:19, 29.60s/it]
 76%|███████▌  | 319/420 [2:38:03<49:51, 29.62s/it]
 76%|███████▌  | 320/420 [2:38:32<49:32, 29.72s/it]
 76%|███████▋  | 321/420 [2:39:02<49:09, 29.80s/it]
 77%|███████▋  | 322/420 [2:39:32<48:36, 29.76s/it]
 77%|███████▋  | 323/420 [2:40:01<47:54, 29.63s/it]
 77%|███████▋  | 324/420 [2:40:31<47:15, 29.54s/it]
 77%|███████▋  | 325/420 [2:41:00<46:40, 29.48s/it]
 78%|███████▊  | 326/420 [2:41:30<46:16, 29.54s/it]
 78%|███████▊  | 327/420 [2:42:00<45:58, 29.67s/it]
 78%|███████▊  | 328/420 [2:42:30<45:39, 29.77s/it]
 78%|███████▊  | 329/420 [2:42:59<44:57, 29.65s/it]
 79%|███████▊  | 330/420 [2:43:29<44:29, 29.66s/it]
 79%|███████▉  | 331/420 [2:43:58<43:56, 29.62s/it]
 79%|███████▉  | 332/420 [2:44:28<43:28, 29.64s/it]
 79%|███████▉  | 333/420 [2:44:58<42:59, 29.65s/it]
 80%|███████▉  | 334/420 [2:45:27<42:31, 29.66s/it]
 80%|███████▉  | 335/420 [2:45:57<41:53, 29.57s/it]
 80%|████████  | 336/420 [2:46:26<41:26, 29.60s/it]
 80%|████████  | 337/420 [2:46:56<40:56, 29.60s/it]
 80%|████████  | 338/420 [2:47:25<40:20, 29.52s/it]
 81%|████████  | 339/420 [2:47:55<39:55, 29.57s/it]
 81%|████████  | 340/420 [2:48:25<39:28, 29.61s/it]
 81%|████████  | 341/420 [2:48:54<39:01, 29.64s/it]
 81%|████████▏ | 342/420 [2:49:24<38:32, 29.65s/it]
 82%|████████▏ | 343/420 [2:49:53<37:55, 29.56s/it]
 82%|████████▏ | 344/420 [2:50:23<37:29, 29.60s/it]
 82%|████████▏ | 345/420 [2:50:53<37:09, 29.72s/it]
 82%|████████▏ | 346/420 [2:51:23<36:38, 29.71s/it]
 83%|████████▎ | 347/420 [2:51:52<36:01, 29.61s/it]
 83%|████████▎ | 348/420 [2:52:22<35:33, 29.63s/it]
 83%|████████▎ | 349/420 [2:52:51<34:56, 29.52s/it]
 83%|████████▎ | 350/420 [2:53:21<34:27, 29.54s/it]
                                                   
+
 83%|████████▎ | 350/420 [2:53:21<34:27, 29.54s/it]
 84%|████████▎ | 351/420 [2:53:51<34:07, 29.67s/it]
 84%|████████▍ | 352/420 [2:54:20<33:38, 29.69s/it]
 84%|████████▍ | 353/420 [2:54:50<33:15, 29.78s/it]
 84%|████████▍ | 354/420 [2:55:20<32:35, 29.63s/it]
 85%|████████▍ | 355/420 [2:55:49<32:07, 29.65s/it]
 85%|████████▍ | 356/420 [2:56:19<31:38, 29.66s/it]
 85%|████████▌ | 357/420 [2:56:49<31:15, 29.77s/it]
 85%|████████▌ | 358/420 [2:57:19<30:38, 29.65s/it]
 85%|████████▌ | 359/420 [2:57:49<30:15, 29.76s/it]
 86%|████████▌ | 360/420 [2:58:19<29:50, 29.84s/it]
 86%|████████▌ | 361/420 [2:58:48<29:17, 29.79s/it]
 86%|████████▌ | 362/420 [2:59:18<28:51, 29.86s/it]
 86%|████████▋ | 363/420 [2:59:48<28:18, 29.81s/it]
 87%|████████▋ | 364/420 [3:00:18<27:47, 29.77s/it]
 87%|████████▋ | 365/420 [3:00:48<27:20, 29.83s/it]
 87%|████████▋ | 366/420 [3:01:17<26:48, 29.79s/it]
 87%|████████▋ | 367/420 [3:01:47<26:16, 29.75s/it]
 88%|████████▊ | 368/420 [3:02:17<25:51, 29.83s/it]
 88%|████████▊ | 369/420 [3:02:47<25:18, 29.78s/it]
 88%|████████▊ | 370/420 [3:03:16<24:42, 29.65s/it]
 88%|████████▊ | 371/420 [3:03:46<24:13, 29.66s/it]
 89%|████████▊ | 372/420 [3:04:15<23:44, 29.67s/it]
 89%|████████▉ | 373/420 [3:04:45<23:09, 29.56s/it]
 89%|████████▉ | 374/420 [3:05:14<22:41, 29.60s/it]
 89%|████████▉ | 375/420 [3:05:44<22:08, 29.52s/it]
 90%|████████▉ | 376/420 [3:06:13<21:41, 29.57s/it]
 90%|████████▉ | 377/420 [3:06:43<21:17, 29.70s/it]
 90%|█████████ | 378/420 [3:07:13<20:51, 29.79s/it]
 90%|█████████ | 379/420 [3:07:43<20:16, 29.66s/it]
 90%|█████████ | 380/420 [3:08:12<19:46, 29.67s/it]
 91%|█████████ | 381/420 [3:08:42<19:21, 29.77s/it]
 91%|█████████ | 382/420 [3:09:12<18:50, 29.75s/it]
 91%|█████████ | 383/420 [3:09:41<18:16, 29.63s/it]
 91%|█████████▏| 384/420 [3:10:11<17:47, 29.64s/it]
 92%|█████████▏| 385/420 [3:10:41<17:21, 29.76s/it]
 92%|█████████▏| 386/420 [3:11:10<16:47, 29.63s/it]
 92%|█████████▏| 387/420 [3:11:40<16:21, 29.73s/it]
 92%|█████████▏| 388/420 [3:12:10<15:53, 29.81s/it]
 93%|█████████▎| 389/420 [3:12:40<15:23, 29.78s/it]
 93%|█████████▎| 390/420 [3:13:10<14:52, 29.75s/it]
 93%|█████████▎| 391/420 [3:13:40<14:24, 29.83s/it]
 93%|█████████▎| 392/420 [3:14:10<13:53, 29.78s/it]
 94%|█████████▎| 393/420 [3:14:39<13:23, 29.75s/it]
 94%|█████████▍| 394/420 [3:15:09<12:53, 29.74s/it]
 94%|█████████▍| 395/420 [3:15:39<12:22, 29.72s/it]
 94%|█████████▍| 396/420 [3:16:08<11:50, 29.61s/it]
 95%|█████████▍| 397/420 [3:16:37<11:18, 29.49s/it]
 95%|█████████▍| 398/420 [3:17:07<10:52, 29.64s/it]
 95%|█████████▌| 399/420 [3:17:37<10:22, 29.66s/it]
 95%|█████████▌| 400/420 [3:18:07<09:53, 29.67s/it]
                                                   
+
 95%|█████████▌| 400/420 [3:18:07<09:53, 29.67s/it]
 95%|█████████▌| 401/420 [3:18:39<09:42, 30.66s/it]
 96%|█████████▌| 402/420 [3:19:09<09:06, 30.36s/it]
 96%|█████████▌| 403/420 [3:19:39<08:30, 30.06s/it]
 96%|█████████▌| 404/420 [3:20:08<07:59, 29.95s/it]
 96%|█████████▋| 405/420 [3:20:38<07:29, 29.97s/it]
 97%|█████████▋| 406/420 [3:21:08<06:59, 29.97s/it]
 97%|█████████▋| 407/420 [3:21:38<06:28, 29.89s/it]
 97%|█████████▋| 408/420 [3:22:08<05:59, 29.92s/it]
 97%|█████████▋| 409/420 [3:22:38<05:28, 29.84s/it]
 98%|█████████▊| 410/420 [3:23:07<04:57, 29.78s/it]
 98%|█████████▊| 411/420 [3:23:37<04:27, 29.75s/it]
 98%|█████████▊| 412/420 [3:24:07<03:57, 29.73s/it]
 98%|█████████▊| 413/420 [3:24:36<03:27, 29.71s/it]
 99%|█████████▊| 414/420 [3:25:06<02:57, 29.59s/it]
 99%|█████████▉| 415/420 [3:25:35<02:28, 29.61s/it]
 99%|█████████▉| 416/420 [3:26:05<01:58, 29.63s/it]
 99%|█████████▉| 417/420 [3:26:34<01:28, 29.53s/it]
                                                   
+
+***** train metrics *****
+  epoch                    =        2.0
+  train_loss               =     0.3738
+  train_runtime            = 3:28:04.14
+  train_samples_per_second =      0.404
+  train_steps_per_second   =      0.034

quality_chunk/train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.37379822503952753,
+    "train_runtime": 12484.1415,
+    "train_samples_per_second": 0.404,
+    "train_steps_per_second": 0.034
+}

quality_chunk/trainer_state.json

@@ -0,0 +1,84 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.995249406175772,
+  "eval_steps": 500,
+  "global_step": 420,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.24,
+      "global_step": 50,
+      "learning_rate": 0.0001,
+      "loss": 0.5683,
+      "step": 50
+    },
+    {
+      "epoch": 0.48,
+      "global_step": 100,
+      "learning_rate": 0.0001,
+      "loss": 0.5393,
+      "step": 100
+    },
+    {
+      "epoch": 0.71,
+      "global_step": 150,
+      "learning_rate": 0.0001,
+      "loss": 0.5376,
+      "step": 150
+    },
+    {
+      "epoch": 0.95,
+      "global_step": 200,
+      "learning_rate": 0.0001,
+      "loss": 0.5331,
+      "step": 200
+    },
+    {
+      "epoch": 1.19,
+      "global_step": 250,
+      "learning_rate": 0.0001,
+      "loss": 0.3011,
+      "step": 250
+    },
+    {
+      "epoch": 1.43,
+      "global_step": 300,
+      "learning_rate": 0.0001,
+      "loss": 0.1997,
+      "step": 300
+    },
+    {
+      "epoch": 1.66,
+      "global_step": 350,
+      "learning_rate": 0.0001,
+      "loss": 0.1919,
+      "step": 350
+    },
+    {
+      "epoch": 1.9,
+      "global_step": 400,
+      "learning_rate": 0.0001,
+      "loss": 0.1817,
+      "step": 400
+    },
+    {
+      "epoch": 2.0,
+      "step": 420,
+      "total_flos": 2.272886224060416e+17,
+      "train_loss": 0.37379822503952753,
+      "train_runtime": 12484.1415,
+      "train_samples_per_second": 0.404,
+      "train_steps_per_second": 0.034
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 420,
+  "num_train_epochs": 2,
+  "save_steps": 200,
+  "total_flos": 2.272886224060416e+17,
+  "trial_name": null,
+  "trial_params": null
+}

quality_random_select/all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.38128303913843065,
+    "train_runtime": 12489.0215,
+    "train_samples_per_second": 0.404,
+    "train_steps_per_second": 0.034
+}

quality_random_select/final/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+### Framework versions
+
+
+- PEFT 0.5.0

quality_random_select/final/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data0/maqi/huggingface_models/option1-models/race_ft",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "o_proj",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

quality_random_select/final/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8655c50b8de5a76320e4c42af57d5d9f0118f4a9c434f10601cc4a4af8449d56
+size 1279424269

quality_random_select/final/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a96acf7897b9680c4f1ec120fc68315a771e900c34f826d7fedf47047b1d6bfb
+size 4091

quality_random_select/quality_random_select.json

@@ -0,0 +1,31 @@
+{
+    "desc": "QuALITY 使用随机选择的关键句训练",
+    "output_dir": "output/ablation_study/QuALITY/quality_random_select",
+    "model_name_or_path": "/data0/maqi/huggingface_models/option1-models/race_ft",
+    "train_file": "/data0/maqi/KGLQA-data/datasets/QuALITY/random_select/quality_random_2048_instruct/train.jsonl",
+    "num_train_epochs": 2,
+    "per_device_train_batch_size": 6,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 1e-4,
+    "max_seq_length": 2048,
+    "logging_steps": 50,
+    "save_steps": 200,
+    "save_total_limit": 1,
+    "lr_scheduler_type": "constant_with_warmup",
+    "warmup_ratio": 0.1,
+    "lora_rank": 128,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+
+    "gradient_checkpointing": true,
+    "disable_tqdm": false,
+    "optim": "paged_adamw_32bit",
+    "seed": 318,
+    "fp16": true,
+    "report_to": "tensorboard",
+    "dataloader_num_workers": 10,
+    "save_strategy": "steps",
+    "weight_decay": 0,
+    "max_grad_norm": 0.3,
+    "remove_unused_columns": false
+}

quality_random_select/train.log

@@ -0,0 +1,32 @@
+model training desc:  QuALITY 使用随机选择的关键句训练
+2023-12-07 17:47:54.813 | INFO     | __main__:init_components:108 - Initializing components...
+
+You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+2023-12-07 17:48:16.828 | INFO     | __main__:init_components:155 - 
+
+2023-12-07 17:48:16.828 | INFO     | __main__:init_components:156 - ********************
+2023-12-07 17:48:16.828 | INFO     | __main__:init_components:157 - using llama2 model
+2023-12-07 17:48:16.828 | INFO     | __main__:init_components:158 - ********************
+2023-12-07 17:48:16.829 | INFO     | __main__:init_components:159 - 
+
+memory footprint of model: 4.024436950683594 GB
+trainable params: 319,815,680 || all params: 7,058,231,296 || trainable%: 4.531102291607305
+2023-12-07 17:49:11.724 | INFO     | component.dataset:__init__:14 - Loading data: /data0/maqi/KGLQA-data/datasets/QuALITY/random_select/quality_random_2048_instruct/train.jsonl
+2023-12-07 17:49:11.842 | INFO     | component.dataset:__init__:19 - there are 2523 data in dataset
+2023-12-07 17:49:11.887 | INFO     | __main__:main:231 - *** starting training ***
+
  0%|          | 0/420 [00:00<?, ?it/s]
  0%|          | 1/420 [00:33<3:51:41, 33.18s/it]
  0%|          | 2/420 [01:02<3:36:16, 31.04s/it]
  1%|          | 3/420 [01:32<3:32:24, 30.56s/it]
  1%|          | 4/420 [02:02<3:29:22, 30.20s/it]
  1%|          | 5/420 [02:32<3:27:37, 30.02s/it]
  1%|▏         | 6/420 [03:02<3:27:08, 30.02s/it]
  2%|▏         | 7/420 [03:31<3:25:56, 29.92s/it]
  2%|▏         | 8/420 [04:01<3:25:01, 29.86s/it]
  2%|▏         | 9/420 [04:31<3:25:00, 29.93s/it]
  2%|▏         | 10/420 [05:01<3:24:48, 29.97s/it]
  3%|▎         | 11/420 [05:31<3:23:44, 29.89s/it]
  3%|▎         | 12/420 [06:01<3:23:37, 29.94s/it]
  3%|▎         | 13/420 [06:31<3:22:42, 29.88s/it]
  3%|▎         | 14/420 [07:00<3:21:06, 29.72s/it]
  4%|▎         | 15/420 [07:29<3:19:50, 29.61s/it]
  4%|▍         | 16/420 [07:59<3:18:49, 29.53s/it]
  4%|▍         | 17/420 [08:29<3:19:25, 29.69s/it]
  4%|▍         | 18/420 [08:58<3:18:55, 29.69s/it]
  5%|▍         | 19/420 [09:29<3:19:12, 29.81s/it]
  5%|▍         | 20/420 [09:58<3:18:26, 29.77s/it]
  5%|▌         | 21/420 [10:28<3:17:44, 29.73s/it]
  5%|▌         | 22/420 [10:58<3:17:09, 29.72s/it]
  5%|▌         | 23/420 [11:27<3:15:56, 29.61s/it]
  6%|▌         | 24/420 [11:57<3:16:21, 29.75s/it]
  6%|▌         | 25/420 [12:27<3:15:47, 29.74s/it]
  6%|▌         | 26/420 [12:57<3:15:56, 29.84s/it]
  6%|▋         | 27/420 [13:27<3:15:55, 29.91s/it]
  7%|▋         | 28/420 [13:57<3:14:59, 29.85s/it]
  7%|▋         | 29/420 [14:26<3:13:32, 29.70s/it]
  7%|▋         | 30/420 [14:56<3:13:44, 29.81s/it]
  7%|▋         | 31/420 [15:26<3:13:05, 29.78s/it]
  8%|▊         | 32/420 [15:55<3:12:29, 29.77s/it]
  8%|▊         | 33/420 [16:26<3:12:33, 29.85s/it]
  8%|▊         | 34/420 [16:55<3:11:44, 29.80s/it]
  8%|▊         | 35/420 [17:25<3:11:44, 29.88s/it]
  9%|▊         | 36/420 [17:55<3:11:34, 29.93s/it]
  9%|▉         | 37/420 [18:25<3:10:35, 29.86s/it]
  9%|▉         | 38/420 [18:55<3:10:31, 29.92s/it]
  9%|▉         | 39/420 [19:25<3:09:36, 29.86s/it]
 10%|▉         | 40/420 [19:55<3:08:51, 29.82s/it]
 10%|▉         | 41/420 [20:24<3:08:03, 29.77s/it]
 10%|█         | 42/420 [20:54<3:08:06, 29.86s/it]
 10%|█         | 43/420 [21:24<3:08:03, 29.93s/it]
 10%|█         | 44/420 [21:54<3:06:28, 29.76s/it]
 11%|█         | 45/420 [22:23<3:05:51, 29.74s/it]
 11%|█         | 46/420 [22:53<3:05:15, 29.72s/it]
 11%|█         | 47/420 [23:23<3:05:22, 29.82s/it]
 11%|█▏        | 48/420 [23:53<3:05:19, 29.89s/it]
 12%|█▏        | 49/420 [24:23<3:04:24, 29.82s/it]
 12%|█▏        | 50/420 [24:52<3:03:02, 29.68s/it]
                                                  
+
 12%|█▏        | 50/420 [24:52<3:03:02, 29.68s/it]
 12%|█▏        | 51/420 [25:22<3:02:28, 29.67s/it]
 12%|█▏        | 52/420 [25:52<3:02:37, 29.78s/it]
 13%|█▎        | 53/420 [26:22<3:02:35, 29.85s/it]
 13%|█▎        | 54/420 [26:52<3:01:49, 29.81s/it]
 13%|█▎        | 55/420 [27:21<3:01:06, 29.77s/it]
 13%|█▎        | 56/420 [27:51<3:01:04, 29.85s/it]
 14%|█▎        | 57/420 [28:21<2:59:42, 29.70s/it]
 14%|█▍        | 58/420 [28:51<2:59:46, 29.80s/it]
 14%|█▍        | 59/420 [29:21<2:59:41, 29.87s/it]
 14%|█▍        | 60/420 [29:50<2:58:52, 29.81s/it]
 15%|█▍        | 61/420 [30:20<2:57:33, 29.67s/it]
 15%|█▍        | 62/420 [30:50<2:57:39, 29.78s/it]
 15%|█▌        | 63/420 [31:20<2:57:38, 29.86s/it]
 15%|█▌        | 64/420 [31:49<2:56:51, 29.81s/it]
 15%|█▌        | 65/420 [32:19<2:56:08, 29.77s/it]
 16%|█▌        | 66/420 [32:49<2:56:05, 29.85s/it]
 16%|█▌        | 67/420 [33:19<2:55:07, 29.76s/it]
 16%|█▌        | 68/420 [33:49<2:55:04, 29.84s/it]
 16%|█▋        | 69/420 [34:18<2:53:42, 29.69s/it]
 17%|█▋        | 70/420 [34:47<2:52:36, 29.59s/it]
 17%|█▋        | 71/420 [35:17<2:52:19, 29.63s/it]
 17%|█▋        | 72/420 [35:47<2:51:21, 29.55s/it]
 17%|█▋        | 73/420 [36:16<2:51:06, 29.59s/it]
 18%|█▊        | 74/420 [36:46<2:50:47, 29.62s/it]
 18%|█▊        | 75/420 [37:16<2:50:15, 29.61s/it]
 18%|█▊        | 76/420 [37:45<2:49:21, 29.54s/it]
 18%|█▊        | 77/420 [38:15<2:49:06, 29.58s/it]
 19%|█▊        | 78/420 [38:45<2:49:22, 29.71s/it]
 19%|█▉        | 79/420 [39:14<2:48:18, 29.61s/it]
 19%|█▉        | 80/420 [39:44<2:48:29, 29.73s/it]
 19%|█▉        | 81/420 [40:13<2:47:21, 29.62s/it]
 20%|█▉        | 82/420 [40:43<2:46:24, 29.54s/it]
 20%|█▉        | 83/420 [41:12<2:46:11, 29.59s/it]
 20%|██        | 84/420 [41:42<2:45:55, 29.63s/it]
 20%|██        | 85/420 [42:12<2:46:04, 29.74s/it]
 20%|██        | 86/420 [42:42<2:45:30, 29.73s/it]
 21%|██        | 87/420 [43:12<2:45:29, 29.82s/it]
 21%|██        | 88/420 [43:42<2:44:48, 29.78s/it]
 21%|██        | 89/420 [44:12<2:44:43, 29.86s/it]
 21%|██▏       | 90/420 [44:42<2:44:29, 29.91s/it]
 22%|██▏       | 91/420 [45:12<2:44:12, 29.95s/it]
 22%|██▏       | 92/420 [45:42<2:43:50, 29.97s/it]
 22%|██▏       | 93/420 [46:11<2:42:53, 29.89s/it]
 22%|██▏       | 94/420 [46:41<2:42:05, 29.83s/it]
 23%|██▎       | 95/420 [47:11<2:41:22, 29.79s/it]
 23%|██▎       | 96/420 [47:41<2:41:16, 29.87s/it]
 23%|██▎       | 97/420 [48:11<2:41:02, 29.92s/it]
 23%|██▎       | 98/420 [48:40<2:39:39, 29.75s/it]
 24%|██▎       | 99/420 [49:10<2:39:37, 29.84s/it]
 24%|██▍       | 100/420 [49:40<2:38:43, 29.76s/it]
                                                   
+
 24%|██▍       | 100/420 [49:40<2:38:43, 29.76s/it]
 24%|██▍       | 101/420 [50:10<2:38:05, 29.74s/it]
 24%|██▍       | 102/420 [50:39<2:37:00, 29.62s/it]
 25%|██▍       | 103/420 [51:08<2:36:06, 29.55s/it]
 25%|██▍       | 104/420 [51:38<2:36:21, 29.69s/it]
 25%|██▌       | 105/420 [52:08<2:35:53, 29.70s/it]
 25%|██▌       | 106/420 [52:38<2:35:23, 29.69s/it]
 25%|██▌       | 107/420 [53:07<2:34:54, 29.69s/it]
 26%|██▌       | 108/420 [53:37<2:33:53, 29.59s/it]
 26%|██▌       | 109/420 [54:07<2:34:02, 29.72s/it]
 26%|██▌       | 110/420 [54:37<2:34:01, 29.81s/it]
 26%|██▋       | 111/420 [55:06<2:32:49, 29.67s/it]
 27%|██▋       | 112/420 [55:35<2:31:51, 29.58s/it]
 27%|██▋       | 113/420 [56:06<2:32:03, 29.72s/it]
 27%|██▋       | 114/420 [56:36<2:32:01, 29.81s/it]
 27%|██▋       | 115/420 [57:05<2:31:21, 29.78s/it]
 28%|██▊       | 116/420 [57:35<2:31:15, 29.85s/it]
 28%|██▊       | 117/420 [58:05<2:31:03, 29.91s/it]
 28%|██▊       | 118/420 [58:35<2:29:44, 29.75s/it]
 28%|██▊       | 119/420 [59:04<2:29:11, 29.74s/it]
 29%|██▊       | 120/420 [59:34<2:28:38, 29.73s/it]
 29%|██▉       | 121/420 [1:00:03<2:27:36, 29.62s/it]
 29%|██▉       | 122/420 [1:00:33<2:26:34, 29.51s/it]
 29%|██▉       | 123/420 [1:01:03<2:26:49, 29.66s/it]
 30%|██▉       | 124/420 [1:01:32<2:26:24, 29.68s/it]
 30%|██▉       | 125/420 [1:02:02<2:26:24, 29.78s/it]
 30%|███       | 126/420 [1:02:33<2:26:16, 29.85s/it]
 30%|███       | 127/420 [1:03:02<2:25:24, 29.78s/it]
 30%|███       | 128/420 [1:03:32<2:24:48, 29.75s/it]
 31%|███       | 129/420 [1:04:01<2:24:12, 29.73s/it]
 31%|███       | 130/420 [1:04:31<2:23:38, 29.72s/it]
 31%|███       | 131/420 [1:05:01<2:23:35, 29.81s/it]
 31%|███▏      | 132/420 [1:05:31<2:23:23, 29.87s/it]
 32%|███▏      | 133/420 [1:06:01<2:22:39, 29.83s/it]
 32%|███▏      | 134/420 [1:06:31<2:22:28, 29.89s/it]
 32%|███▏      | 135/420 [1:07:01<2:21:41, 29.83s/it]
 32%|███▏      | 136/420 [1:07:30<2:20:33, 29.69s/it]
 33%|███▎      | 137/420 [1:08:00<2:20:04, 29.70s/it]
 33%|███▎      | 138/420 [1:08:30<2:20:04, 29.80s/it]
 33%|███▎      | 139/420 [1:09:00<2:19:54, 29.87s/it]
 33%|███▎      | 140/420 [1:09:30<2:19:11, 29.83s/it]
 34%|███▎      | 141/420 [1:09:59<2:18:31, 29.79s/it]
 34%|███▍      | 142/420 [1:10:29<2:17:26, 29.66s/it]
 34%|███▍      | 143/420 [1:10:58<2:16:59, 29.67s/it]
 34%|███▍      | 144/420 [1:11:28<2:16:05, 29.58s/it]
 35%|███▍      | 145/420 [1:11:57<2:15:44, 29.62s/it]
 35%|███▍      | 146/420 [1:12:27<2:15:21, 29.64s/it]
 35%|███▌      | 147/420 [1:12:57<2:14:57, 29.66s/it]
 35%|███▌      | 148/420 [1:13:27<2:14:57, 29.77s/it]
 35%|███▌      | 149/420 [1:13:56<2:13:46, 29.62s/it]
 36%|███▌      | 150/420 [1:14:25<2:12:56, 29.54s/it]
                                                     
+
 36%|███▌      | 150/420 [1:14:25<2:12:56, 29.54s/it]
 36%|███▌      | 151/420 [1:14:55<2:12:40, 29.59s/it]
 36%|███▌      | 152/420 [1:15:25<2:12:17, 29.62s/it]
 36%|███▋      | 153/420 [1:15:55<2:11:54, 29.64s/it]
 37%|███▋      | 154/420 [1:16:25<2:11:55, 29.76s/it]
 37%|███▋      | 155/420 [1:16:54<2:11:21, 29.74s/it]
 37%|███▋      | 156/420 [1:17:24<2:10:22, 29.63s/it]
 37%|███▋      | 157/420 [1:17:53<2:09:56, 29.65s/it]
 38%|███▊      | 158/420 [1:18:23<2:09:05, 29.56s/it]
 38%|███▊      | 159/420 [1:18:53<2:09:12, 29.70s/it]
 38%|███▊      | 160/420 [1:19:22<2:08:42, 29.70s/it]
 38%|███▊      | 161/420 [1:19:52<2:08:37, 29.80s/it]
 39%|███▊      | 162/420 [1:20:22<2:08:25, 29.87s/it]
 39%|███▉      | 163/420 [1:20:52<2:07:41, 29.81s/it]
 39%|███▉      | 164/420 [1:21:22<2:07:27, 29.87s/it]
 39%|███▉      | 165/420 [1:21:52<2:06:18, 29.72s/it]
 40%|███▉      | 166/420 [1:22:21<2:05:47, 29.72s/it]
 40%|███▉      | 167/420 [1:22:51<2:05:16, 29.71s/it]
 40%|████      | 168/420 [1:23:21<2:04:45, 29.71s/it]
 40%|████      | 169/420 [1:23:50<2:04:15, 29.70s/it]
 40%|████      | 170/420 [1:24:20<2:03:44, 29.70s/it]
 41%|████      | 171/420 [1:24:50<2:03:13, 29.69s/it]
 41%|████      | 172/420 [1:25:20<2:03:08, 29.79s/it]
 41%|████      | 173/420 [1:25:49<2:02:30, 29.76s/it]
 41%|████▏     | 174/420 [1:26:19<2:01:57, 29.75s/it]
 42%|████▏     | 175/420 [1:26:49<2:01:22, 29.73s/it]
 42%|████▏     | 176/420 [1:27:18<2:00:25, 29.61s/it]
 42%|████▏     | 177/420 [1:27:48<2:00:00, 29.63s/it]
 42%|████▏     | 178/420 [1:28:18<1:59:35, 29.65s/it]
 43%|████▎     | 179/420 [1:28:47<1:59:09, 29.67s/it]
 43%|████▎     | 180/420 [1:29:17<1:58:42, 29.68s/it]
 43%|████▎     | 181/420 [1:29:47<1:58:38, 29.78s/it]
 43%|████▎     | 182/420 [1:30:17<1:58:25, 29.85s/it]
 44%|████▎     | 183/420 [1:30:47<1:57:44, 29.81s/it]
 44%|████▍     | 184/420 [1:31:16<1:57:07, 29.78s/it]
 44%|████▍     | 185/420 [1:31:46<1:56:33, 29.76s/it]
 44%|████▍     | 186/420 [1:32:16<1:56:00, 29.75s/it]
 45%|████▍     | 187/420 [1:32:46<1:55:29, 29.74s/it]
 45%|████▍     | 188/420 [1:33:15<1:54:59, 29.74s/it]
 45%|████▌     | 189/420 [1:33:45<1:54:50, 29.83s/it]
 45%|████▌     | 190/420 [1:34:15<1:54:11, 29.79s/it]
 45%|████▌     | 191/420 [1:34:45<1:53:36, 29.77s/it]
 46%|████▌     | 192/420 [1:35:14<1:53:01, 29.75s/it]
 46%|████▌     | 193/420 [1:35:44<1:52:29, 29.74s/it]
 46%|████▌     | 194/420 [1:36:14<1:51:35, 29.63s/it]
 46%|████▋     | 195/420 [1:36:43<1:50:48, 29.55s/it]
 47%|████▋     | 196/420 [1:37:13<1:50:28, 29.59s/it]
 47%|████▋     | 197/420 [1:37:43<1:50:26, 29.72s/it]
 47%|████▋     | 198/420 [1:38:12<1:49:55, 29.71s/it]
 47%|████▋     | 199/420 [1:38:42<1:49:19, 29.68s/it]
 48%|████▊     | 200/420 [1:39:12<1:49:13, 29.79s/it]
                                                     
+
 48%|████▊     | 200/420 [1:39:12<1:49:13, 29.79s/it]
 48%|████▊     | 201/420 [1:39:46<1:53:23, 31.07s/it]
 48%|████▊     | 202/420 [1:40:16<1:51:23, 30.66s/it]
 48%|████▊     | 203/420 [1:40:46<1:50:13, 30.47s/it]
 49%|████▊     | 204/420 [1:41:15<1:48:46, 30.22s/it]
 49%|████▉     | 205/420 [1:41:45<1:47:43, 30.06s/it]
 49%|████▉     | 206/420 [1:42:15<1:46:51, 29.96s/it]
 49%|████▉     | 207/420 [1:42:44<1:45:43, 29.78s/it]
 50%|████▉     | 208/420 [1:43:14<1:45:30, 29.86s/it]
 50%|████▉     | 209/420 [1:43:44<1:44:51, 29.82s/it]
 50%|█████     | 210/420 [1:44:14<1:44:34, 29.88s/it]
 50%|█████     | 211/420 [1:44:39<1:38:39, 28.32s/it]
 50%|█████     | 212/420 [1:45:08<1:39:37, 28.74s/it]
 51%|█████     | 213/420 [1:45:38<1:40:30, 29.13s/it]
 51%|█████     | 214/420 [1:46:08<1:40:35, 29.30s/it]
 51%|█████     | 215/420 [1:46:38<1:40:50, 29.51s/it]
 51%|█████▏    | 216/420 [1:47:07<1:40:06, 29.44s/it]
 52%|█████▏    | 217/420 [1:47:37<1:39:52, 29.52s/it]
 52%|█████▏    | 218/420 [1:48:07<1:39:33, 29.57s/it]
 52%|█████▏    | 219/420 [1:48:36<1:39:12, 29.61s/it]
 52%|█████▏    | 220/420 [1:49:06<1:38:28, 29.54s/it]
 53%|█████▎    | 221/420 [1:49:36<1:38:07, 29.58s/it]
 53%|█████▎    | 222/420 [1:50:06<1:38:02, 29.71s/it]
 53%|█████▎    | 223/420 [1:50:36<1:37:51, 29.81s/it]
 53%|█████▎    | 224/420 [1:51:05<1:36:55, 29.67s/it]
 54%|█████▎    | 225/420 [1:51:35<1:36:28, 29.68s/it]
 54%|█████▍    | 226/420 [1:52:04<1:35:59, 29.69s/it]
 54%|█████▍    | 227/420 [1:52:34<1:35:28, 29.68s/it]
 54%|█████▍    | 228/420 [1:53:04<1:34:58, 29.68s/it]
 55%|█████▍    | 229/420 [1:53:34<1:34:48, 29.78s/it]
 55%|█████▍    | 230/420 [1:54:04<1:34:32, 29.86s/it]
 55%|█████▌    | 231/420 [1:54:33<1:33:53, 29.80s/it]
 55%|█████▌    | 232/420 [1:55:03<1:32:58, 29.67s/it]
 55%|█████▌    | 233/420 [1:55:32<1:32:29, 29.68s/it]
 56%|█████▌    | 234/420 [1:56:02<1:32:17, 29.77s/it]
 56%|█████▌    | 235/420 [1:56:32<1:32:01, 29.85s/it]
 56%|█████▌    | 236/420 [1:57:02<1:31:24, 29.81s/it]
 56%|█████▋    | 237/420 [1:57:32<1:31:06, 29.87s/it]
 57%|█████▋    | 238/420 [1:58:02<1:30:09, 29.72s/it]
 57%|█████▋    | 239/420 [1:58:32<1:29:56, 29.82s/it]
 57%|█████▋    | 240/420 [1:59:01<1:29:01, 29.68s/it]
 57%|█████▋    | 241/420 [1:59:31<1:28:32, 29.68s/it]
 58%|█████▊    | 242/420 [2:00:00<1:27:46, 29.59s/it]
 58%|█████▊    | 243/420 [2:00:29<1:27:04, 29.52s/it]
 58%|█████▊    | 244/420 [2:00:59<1:27:01, 29.66s/it]
 58%|█████▊    | 245/420 [2:01:29<1:26:15, 29.58s/it]
 59%|█████▊    | 246/420 [2:01:58<1:25:51, 29.60s/it]
 59%|█████▉    | 247/420 [2:02:28<1:25:07, 29.52s/it]
 59%|█████▉    | 248/420 [2:02:57<1:24:28, 29.47s/it]
 59%|█████▉    | 249/420 [2:03:27<1:24:27, 29.63s/it]
 60%|█████▉    | 250/420 [2:03:56<1:23:44, 29.56s/it]
                                                     
+
 60%|█████▉    | 250/420 [2:03:56<1:23:44, 29.56s/it]
 60%|█████▉    | 251/420 [2:04:26<1:23:36, 29.69s/it]
 60%|██████    | 252/420 [2:04:56<1:22:50, 29.58s/it]
 60%|██████    | 253/420 [2:05:25<1:22:25, 29.62s/it]
 60%|██████    | 254/420 [2:05:55<1:21:42, 29.53s/it]
 61%|██████    | 255/420 [2:06:25<1:21:36, 29.68s/it]
 61%|██████    | 256/420 [2:06:55<1:21:07, 29.68s/it]
 61%|██████    | 257/420 [2:07:24<1:20:33, 29.65s/it]
 61%|██████▏   | 258/420 [2:07:54<1:20:20, 29.76s/it]
 62%|██████▏   | 259/420 [2:08:23<1:19:30, 29.63s/it]
 62%|██████▏   | 260/420 [2:08:53<1:19:03, 29.65s/it]
 62%|██████▏   | 261/420 [2:09:23<1:18:49, 29.74s/it]
 62%|██████▏   | 262/420 [2:09:53<1:18:16, 29.72s/it]
 63%|██████▎   | 263/420 [2:10:22<1:17:44, 29.71s/it]
 63%|██████▎   | 264/420 [2:10:52<1:17:13, 29.70s/it]
 63%|██████▎   | 265/420 [2:11:21<1:16:26, 29.59s/it]
 63%|██████▎   | 266/420 [2:11:51<1:16:00, 29.61s/it]
 64%|██████▎   | 267/420 [2:12:21<1:15:18, 29.53s/it]
 64%|██████▍   | 268/420 [2:12:50<1:14:40, 29.47s/it]
 64%|██████▍   | 269/420 [2:13:20<1:14:35, 29.64s/it]
 64%|██████▍   | 270/420 [2:13:50<1:14:08, 29.65s/it]
 65%|██████▍   | 271/420 [2:14:20<1:13:54, 29.76s/it]
 65%|██████▍   | 272/420 [2:14:49<1:13:16, 29.71s/it]
 65%|██████▌   | 273/420 [2:15:18<1:12:31, 29.60s/it]
 65%|██████▌   | 274/420 [2:15:48<1:12:05, 29.62s/it]
 65%|██████▌   | 275/420 [2:16:18<1:11:37, 29.64s/it]
 66%|██████▌   | 276/420 [2:16:47<1:10:55, 29.55s/it]
 66%|██████▌   | 277/420 [2:17:17<1:10:17, 29.50s/it]
 66%|██████▌   | 278/420 [2:17:47<1:10:08, 29.64s/it]
 66%|██████▋   | 279/420 [2:18:16<1:09:40, 29.65s/it]
 67%|██████▋   | 280/420 [2:18:46<1:09:11, 29.65s/it]
 67%|██████▋   | 281/420 [2:19:15<1:08:28, 29.56s/it]
 67%|██████▋   | 282/420 [2:19:45<1:07:59, 29.56s/it]
 67%|██████▋   | 283/420 [2:20:14<1:07:21, 29.50s/it]
 68%|██████▊   | 284/420 [2:20:44<1:07:12, 29.65s/it]
 68%|██████▊   | 285/420 [2:21:14<1:06:57, 29.76s/it]
 68%|██████▊   | 286/420 [2:21:44<1:06:37, 29.84s/it]
 68%|██████▊   | 287/420 [2:22:14<1:06:14, 29.88s/it]
 69%|██████▊   | 288/420 [2:22:44<1:05:35, 29.82s/it]
 69%|██████▉   | 289/420 [2:23:13<1:05:00, 29.77s/it]
 69%|██████▉   | 290/420 [2:23:43<1:04:25, 29.74s/it]
 69%|██████▉   | 291/420 [2:24:12<1:03:40, 29.62s/it]
 70%|██████▉   | 292/420 [2:24:42<1:03:00, 29.54s/it]
 70%|██████▉   | 293/420 [2:25:11<1:02:36, 29.58s/it]
 70%|███████   | 294/420 [2:25:41<1:02:10, 29.61s/it]
 70%|███████   | 295/420 [2:26:11<1:01:55, 29.72s/it]
 70%|███████   | 296/420 [2:26:41<1:01:22, 29.70s/it]
 71%|███████   | 297/420 [2:27:11<1:01:04, 29.79s/it]
 71%|███████   | 298/420 [2:27:41<1:00:30, 29.76s/it]
 71%|███████   | 299/420 [2:28:10<59:58, 29.74s/it]  
 71%|███████▏  | 300/420 [2:28:40<59:15, 29.63s/it]
                                                   
+
 71%|███████▏  | 300/420 [2:28:40<59:15, 29.63s/it]
 72%|███████▏  | 301/420 [2:29:10<58:58, 29.74s/it]
 72%|███████▏  | 302/420 [2:29:40<58:37, 29.81s/it]
 72%|███████▏  | 303/420 [2:30:09<58:02, 29.76s/it]
 72%|███████▏  | 304/420 [2:30:39<57:17, 29.63s/it]
 73%|███████▎  | 305/420 [2:31:08<56:48, 29.64s/it]
 73%|███████▎  | 306/420 [2:31:38<56:31, 29.75s/it]
 73%|███████▎  | 307/420 [2:32:08<55:58, 29.72s/it]
 73%|███████▎  | 308/420 [2:32:37<55:26, 29.70s/it]
 74%|███████▎  | 309/420 [2:33:07<54:55, 29.69s/it]
 74%|███████▍  | 310/420 [2:33:37<54:24, 29.68s/it]
 74%|███████▍  | 311/420 [2:34:06<53:44, 29.58s/it]
 74%|███████▍  | 312/420 [2:34:36<53:17, 29.61s/it]
 75%|███████▍  | 313/420 [2:35:06<53:00, 29.72s/it]
 75%|███████▍  | 314/420 [2:35:35<52:28, 29.70s/it]
 75%|███████▌  | 315/420 [2:36:05<51:58, 29.70s/it]
 75%|███████▌  | 316/420 [2:36:35<51:37, 29.78s/it]
 75%|███████▌  | 317/420 [2:37:05<51:14, 29.85s/it]
 76%|███████▌  | 318/420 [2:37:35<50:38, 29.79s/it]
 76%|███████▌  | 319/420 [2:38:04<50:04, 29.75s/it]
 76%|███████▌  | 320/420 [2:38:34<49:42, 29.82s/it]
 76%|███████▋  | 321/420 [2:39:04<49:08, 29.78s/it]
 77%|███████▋  | 322/420 [2:39:33<48:25, 29.65s/it]
 77%|███████▋  | 323/420 [2:40:03<48:05, 29.75s/it]
 77%|███████▋  | 324/420 [2:40:33<47:43, 29.83s/it]
 77%|███████▋  | 325/420 [2:41:03<47:18, 29.88s/it]
 78%|███████▊  | 326/420 [2:41:33<46:42, 29.82s/it]
 78%|███████▊  | 327/420 [2:42:03<46:08, 29.77s/it]
 78%|███████▊  | 328/420 [2:42:32<45:27, 29.65s/it]
 78%|███████▊  | 329/420 [2:43:02<45:08, 29.76s/it]
 79%|███████▊  | 330/420 [2:43:32<44:36, 29.73s/it]
 79%|███████▉  | 331/420 [2:44:02<44:04, 29.72s/it]
 79%|███████▉  | 332/420 [2:44:32<43:42, 29.80s/it]
 79%|███████▉  | 333/420 [2:45:02<43:18, 29.86s/it]
 80%|███████▉  | 334/420 [2:45:31<42:43, 29.81s/it]
 80%|███████▉  | 335/420 [2:46:01<42:10, 29.77s/it]
 80%|████████  | 336/420 [2:46:30<41:30, 29.64s/it]
 80%|████████  | 337/420 [2:47:00<41:09, 29.75s/it]
 80%|████████  | 338/420 [2:47:30<40:37, 29.73s/it]
 81%|████████  | 339/420 [2:48:00<40:06, 29.71s/it]
 81%|████████  | 340/420 [2:48:29<39:35, 29.69s/it]
 81%|████████  | 341/420 [2:48:59<39:12, 29.78s/it]
 81%|████████▏ | 342/420 [2:49:29<38:48, 29.85s/it]
 82%|████████▏ | 343/420 [2:49:59<38:13, 29.79s/it]
 82%|████████▏ | 344/420 [2:50:29<37:41, 29.76s/it]
 82%|████████▏ | 345/420 [2:50:59<37:17, 29.83s/it]
 82%|████████▏ | 346/420 [2:51:28<36:43, 29.78s/it]
 83%|████████▎ | 347/420 [2:51:58<36:18, 29.84s/it]
 83%|████████▎ | 348/420 [2:52:28<35:37, 29.69s/it]
 83%|████████▎ | 349/420 [2:52:58<35:14, 29.78s/it]
 83%|████████▎ | 350/420 [2:53:27<34:42, 29.75s/it]
                                                   
+
 83%|████████▎ | 350/420 [2:53:27<34:42, 29.75s/it]
 84%|████████▎ | 351/420 [2:53:57<34:10, 29.71s/it]
 84%|████████▍ | 352/420 [2:54:26<33:39, 29.69s/it]
 84%|████████▍ | 353/420 [2:54:56<33:14, 29.78s/it]
 84%|████████▍ | 354/420 [2:55:26<32:42, 29.73s/it]
 85%|████████▍ | 355/420 [2:55:56<32:10, 29.71s/it]
 85%|████████▍ | 356/420 [2:56:25<31:40, 29.70s/it]
 85%|████████▌ | 357/420 [2:56:55<31:03, 29.58s/it]
 85%|████████▌ | 358/420 [2:57:24<30:33, 29.57s/it]
 85%|████████▌ | 359/420 [2:57:54<30:05, 29.59s/it]
 86%|████████▌ | 360/420 [2:58:24<29:36, 29.61s/it]
 86%|████████▌ | 361/420 [2:58:54<29:13, 29.72s/it]
 86%|████████▌ | 362/420 [2:59:24<28:48, 29.80s/it]
 86%|████████▋ | 363/420 [2:59:53<28:14, 29.72s/it]
 87%|████████▋ | 364/420 [3:00:23<27:43, 29.70s/it]
 87%|████████▋ | 365/420 [3:00:53<27:18, 29.79s/it]
 87%|████████▋ | 366/420 [3:01:22<26:40, 29.64s/it]
 87%|████████▋ | 367/420 [3:01:52<26:11, 29.64s/it]
 88%|████████▊ | 368/420 [3:02:21<25:40, 29.62s/it]
 88%|████████▊ | 369/420 [3:02:51<25:10, 29.63s/it]
 88%|████████▊ | 370/420 [3:03:21<24:46, 29.73s/it]
 88%|████████▊ | 371/420 [3:03:51<24:16, 29.72s/it]
 89%|████████▊ | 372/420 [3:04:20<23:40, 29.60s/it]
 89%|████████▉ | 373/420 [3:04:49<23:07, 29.52s/it]
 89%|████████▉ | 374/420 [3:05:19<22:39, 29.56s/it]
 89%|████████▉ | 375/420 [3:05:49<22:16, 29.69s/it]
 90%|████████▉ | 376/420 [3:06:19<21:50, 29.78s/it]
 90%|████████▉ | 377/420 [3:06:48<21:18, 29.74s/it]
 90%|█████████ | 378/420 [3:07:18<20:48, 29.72s/it]
 90%|█████████ | 379/420 [3:07:47<20:13, 29.60s/it]
 90%|█████████ | 380/420 [3:08:17<19:40, 29.52s/it]
 91%|█████████ | 381/420 [3:08:46<19:09, 29.46s/it]
 91%|█████████ | 382/420 [3:09:16<18:45, 29.61s/it]
 91%|█████████ | 383/420 [3:09:45<18:12, 29.53s/it]
 91%|█████████▏| 384/420 [3:10:15<17:44, 29.56s/it]
 92%|█████████▏| 385/420 [3:10:44<17:12, 29.49s/it]
 92%|█████████▏| 386/420 [3:11:14<16:47, 29.63s/it]
 92%|█████████▏| 387/420 [3:11:44<16:21, 29.73s/it]
 92%|█████████▏| 388/420 [3:12:14<15:50, 29.71s/it]
 93%|█████████▎| 389/420 [3:12:44<15:20, 29.69s/it]
 93%|█████████▎| 390/420 [3:13:14<14:53, 29.78s/it]
 93%|█████████▎| 391/420 [3:13:43<14:19, 29.64s/it]
 93%|█████████▎| 392/420 [3:14:13<13:52, 29.74s/it]
 94%|█████████▎| 393/420 [3:14:43<13:24, 29.81s/it]
 94%|█████████▍| 394/420 [3:15:13<12:53, 29.76s/it]
 94%|█████████▍| 395/420 [3:15:42<12:23, 29.74s/it]
 94%|█████████▍| 396/420 [3:16:12<11:53, 29.71s/it]
 95%|█████████▍| 397/420 [3:16:42<11:25, 29.80s/it]
 95%|█████████▍| 398/420 [3:17:11<10:54, 29.75s/it]
 95%|█████████▌| 399/420 [3:17:41<10:26, 29.83s/it]
 95%|█████████▌| 400/420 [3:18:11<09:57, 29.87s/it]
                                                   
+
 95%|█████████▌| 400/420 [3:18:11<09:57, 29.87s/it]
 95%|█████████▌| 401/420 [3:18:45<09:47, 30.94s/it]
 96%|█████████▌| 402/420 [3:19:15<09:11, 30.66s/it]
 96%|█████████▌| 403/420 [3:19:44<08:34, 30.26s/it]
 96%|█████████▌| 404/420 [3:20:14<07:59, 29.98s/it]
 96%|█████████▋| 405/420 [3:20:43<07:28, 29.89s/it]
 97%|█████████▋| 406/420 [3:21:13<06:56, 29.73s/it]
 97%|█████████▋| 407/420 [3:21:42<06:25, 29.67s/it]
 97%|█████████▋| 408/420 [3:22:11<05:54, 29.57s/it]
 97%|█████████▋| 409/420 [3:22:41<05:24, 29.50s/it]
 98%|█████████▊| 410/420 [3:23:10<04:55, 29.55s/it]
 98%|█████████▊| 411/420 [3:23:40<04:27, 29.68s/it]
 98%|█████████▊| 412/420 [3:24:10<03:58, 29.77s/it]
 98%|█████████▊| 413/420 [3:24:40<03:28, 29.74s/it]
 99%|█████████▊| 414/420 [3:25:10<02:58, 29.82s/it]
 99%|█████████▉| 415/420 [3:25:40<02:28, 29.77s/it]
 99%|█████████▉| 416/420 [3:26:09<01:58, 29.64s/it]
 99%|█████████▉| 417/420 [3:26:39<01:28, 29.64s/it]
                                                   
+
+***** train metrics *****
+  epoch                    =        2.0
+  train_loss               =     0.3813
+  train_runtime            = 3:28:09.02
+  train_samples_per_second =      0.404
+  train_steps_per_second   =      0.034

quality_random_select/train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.38128303913843065,
+    "train_runtime": 12489.0215,
+    "train_samples_per_second": 0.404,
+    "train_steps_per_second": 0.034
+}

quality_random_select/trainer_state.json

@@ -0,0 +1,84 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.995249406175772,
+  "eval_steps": 500,
+  "global_step": 420,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.24,
+      "global_step": 50,
+      "learning_rate": 0.0001,
+      "loss": 0.5905,
+      "step": 50
+    },
+    {
+      "epoch": 0.48,
+      "global_step": 100,
+      "learning_rate": 0.0001,
+      "loss": 0.5458,
+      "step": 100
+    },
+    {
+      "epoch": 0.71,
+      "global_step": 150,
+      "learning_rate": 0.0001,
+      "loss": 0.532,
+      "step": 150
+    },
+    {
+      "epoch": 0.95,
+      "global_step": 200,
+      "learning_rate": 0.0001,
+      "loss": 0.5351,
+      "step": 200
+    },
+    {
+      "epoch": 1.19,
+      "global_step": 250,
+      "learning_rate": 0.0001,
+      "loss": 0.2802,
+      "step": 250
+    },
+    {
+      "epoch": 1.43,
+      "global_step": 300,
+      "learning_rate": 0.0001,
+      "loss": 0.2154,
+      "step": 300
+    },
+    {
+      "epoch": 1.66,
+      "global_step": 350,
+      "learning_rate": 0.0001,
+      "loss": 0.2142,
+      "step": 350
+    },
+    {
+      "epoch": 1.9,
+      "global_step": 400,
+      "learning_rate": 0.0001,
+      "loss": 0.2111,
+      "step": 400
+    },
+    {
+      "epoch": 2.0,
+      "step": 420,
+      "total_flos": 2.2729114462558618e+17,
+      "train_loss": 0.38128303913843065,
+      "train_runtime": 12489.0215,
+      "train_samples_per_second": 0.404,
+      "train_steps_per_second": 0.034
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 420,
+  "num_train_epochs": 2,
+  "save_steps": 200,
+  "total_flos": 2.2729114462558618e+17,
+  "trial_name": null,
+  "trial_params": null
+}

without_ncr_ft/all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.9344269825124193,
+    "train_runtime": 12670.7287,
+    "train_samples_per_second": 0.822,
+    "train_steps_per_second": 0.041
+}

without_ncr_ft/cclue_ft.json

@@ -0,0 +1,31 @@
+{
+    "desc": "CCLUE-MRC仅在TechGPT-7B上训练",
+    "output_dir": "output/ablation_study/CCLUE/without_ncr_ft",
+    "model_name_or_path": "/data0/maqi/huggingface_models/TechGPT-7B",
+    "train_file": "/data0/maqi/KGLQA-data/datasets/CCLUE/cclue_instruct/train.jsonl",
+    "num_train_epochs": 3,
+    "per_device_train_batch_size": 10,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 1e-4,
+    "max_seq_length": 1400,
+    "logging_steps": 20,
+    "save_steps": 200,
+    "save_total_limit": 1,
+    "lr_scheduler_type": "constant_with_warmup",
+    "warmup_ratio": 0.1,
+    "lora_rank": 128,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+
+    "gradient_checkpointing": true,
+    "disable_tqdm": false,
+    "optim": "paged_adamw_32bit",
+    "seed": 318,
+    "fp16": true,
+    "report_to": "tensorboard",
+    "dataloader_num_workers": 10,
+    "save_strategy": "steps",
+    "weight_decay": 0,
+    "max_grad_norm": 0.3,
+    "remove_unused_columns": false
+}

without_ncr_ft/final/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+### Framework versions
+
+
+- PEFT 0.5.0

without_ncr_ft/final/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data0/maqi/huggingface_models/TechGPT-7B",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "o_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

without_ncr_ft/final/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2b37b656a07274212eaeaaae43ca8baab734dbc61c174696a785d4407de7fa9
+size 1279424269

without_ncr_ft/final/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d905bc24ad6d106cecc0ba4a7b39239eb4b3fb6ce7db2fe82f8a70de6ec1ccd
+size 4091

without_ncr_ft/train.log

@@ -0,0 +1,49 @@
+model training desc:  CCLUE-MRC仅在TechGPT-7B上训练
+2023-12-04 14:37:55.661 | INFO     | __main__:init_components:108 - Initializing components...
+You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+2023-12-04 14:38:18.229 | INFO     | __main__:init_components:143 - 
+
+2023-12-04 14:38:18.229 | INFO     | __main__:init_components:144 - ********************
+2023-12-04 14:38:18.229 | INFO     | __main__:init_components:145 - using TechGPT-7B
+2023-12-04 14:38:18.229 | INFO     | __main__:init_components:146 - ********************
+2023-12-04 14:38:18.229 | INFO     | __main__:init_components:147 - 
+
+memory footprint of model: 5.472740173339844 GB
+trainable params: 319,815,680 || all params: 7,447,007,232 || trainable%: 4.294553100818044
+2023-12-04 14:39:00.475 | INFO     | component.dataset:__init__:14 - Loading data: /data0/maqi/KGLQA-data/datasets/CCLUE/cclue_instruct/train.jsonl
+2023-12-04 14:39:00.498 | INFO     | component.dataset:__init__:19 - there are 3473 data in dataset
+2023-12-04 14:39:00.535 | INFO     | __main__:main:231 - *** starting training ***
+
  0%|          | 0/522 [00:00<?, ?it/s]
  0%|          | 1/522 [00:24<3:34:55, 24.75s/it]
  0%|          | 2/522 [00:50<3:38:56, 25.26s/it]
  1%|          | 3/522 [01:13<3:31:10, 24.41s/it]
  1%|          | 4/522 [01:42<3:45:26, 26.11s/it]
  1%|          | 5/522 [02:09<3:47:32, 26.41s/it]
  1%|          | 6/522 [02:37<3:51:28, 26.92s/it]
  1%|▏         | 7/522 [03:03<3:48:31, 26.62s/it]
  2%|▏         | 8/522 [03:30<3:48:53, 26.72s/it]
  2%|▏         | 9/522 [03:57<3:50:43, 26.98s/it]
  2%|▏         | 10/522 [04:20<3:38:15, 25.58s/it]
  2%|▏         | 11/522 [04:46<3:39:12, 25.74s/it]
  2%|▏         | 12/522 [05:04<3:20:10, 23.55s/it]
  2%|▏         | 13/522 [05:33<3:31:54, 24.98s/it]
  3%|▎         | 14/522 [05:51<3:14:44, 23.00s/it]
  3%|▎         | 15/522 [06:18<3:23:20, 24.06s/it]
  3%|▎         | 16/522 [06:41<3:21:27, 23.89s/it]
  3%|▎         | 17/522 [07:07<3:24:51, 24.34s/it]
  3%|▎         | 18/522 [07:31<3:24:56, 24.40s/it]
  4%|▎         | 19/522 [07:56<3:25:54, 24.56s/it]
  4%|▍         | 20/522 [08:20<3:25:05, 24.51s/it]
                                                  
+
  4%|▍         | 20/522 [08:20<3:25:05, 24.51s/it]
  4%|▍         | 21/522 [08:47<3:29:46, 25.12s/it]
  4%|▍         | 22/522 [09:08<3:19:00, 23.88s/it]
  4%|▍         | 23/522 [09:39<3:35:20, 25.89s/it]
  5%|▍         | 24/522 [10:01<3:27:38, 25.02s/it]
  5%|▍         | 25/522 [10:28<3:30:10, 25.37s/it]
  5%|▍         | 26/522 [10:53<3:30:29, 25.46s/it]
  5%|▌         | 27/522 [11:19<3:30:50, 25.56s/it]
  5%|▌         | 28/522 [11:42<3:23:28, 24.71s/it]
  6%|▌         | 29/522 [12:02<3:11:09, 23.26s/it]
  6%|▌         | 30/522 [12:21<3:00:56, 22.07s/it]
  6%|▌         | 31/522 [12:47<3:11:01, 23.34s/it]
  6%|▌         | 32/522 [13:09<3:05:17, 22.69s/it]
  6%|▋         | 33/522 [13:32<3:07:37, 23.02s/it]
  7%|▋         | 34/522 [13:53<3:01:04, 22.26s/it]
  7%|▋         | 35/522 [14:19<3:10:33, 23.48s/it]
  7%|▋         | 36/522 [14:44<3:14:20, 23.99s/it]
  7%|▋         | 37/522 [15:11<3:21:33, 24.94s/it]
  7%|▋         | 38/522 [15:36<3:19:47, 24.77s/it]
  7%|▋         | 39/522 [16:01<3:20:47, 24.94s/it]
  8%|▊         | 40/522 [16:29<3:28:12, 25.92s/it]
                                                  
+
  8%|▊         | 40/522 [16:29<3:28:12, 25.92s/it]
  8%|▊         | 41/522 [16:51<3:17:33, 24.64s/it]
  8%|▊         | 42/522 [17:16<3:17:28, 24.68s/it]
  8%|▊         | 43/522 [17:42<3:20:12, 25.08s/it]
  8%|▊         | 44/522 [18:09<3:23:42, 25.57s/it]
  9%|▊         | 45/522 [18:34<3:22:32, 25.48s/it]
  9%|▉         | 46/522 [18:58<3:20:09, 25.23s/it]
  9%|▉         | 47/522 [19:24<3:19:59, 25.26s/it]
  9%|▉         | 48/522 [19:49<3:20:26, 25.37s/it]
  9%|▉         | 49/522 [20:11<3:10:43, 24.19s/it]
 10%|▉         | 50/522 [20:39<3:20:13, 25.45s/it]
 10%|▉         | 51/522 [21:02<3:13:58, 24.71s/it]
 10%|▉         | 52/522 [21:27<3:12:49, 24.62s/it]
 10%|█         | 53/522 [21:50<3:09:26, 24.23s/it]
 10%|█         | 54/522 [22:09<2:57:16, 22.73s/it]
 11%|█         | 55/522 [22:34<3:00:44, 23.22s/it]
 11%|█         | 56/522 [22:57<3:00:14, 23.21s/it]
 11%|█         | 57/522 [23:19<2:56:51, 22.82s/it]
 11%|█         | 58/522 [23:40<2:52:06, 22.26s/it]
 11%|█▏        | 59/522 [24:01<2:49:34, 21.97s/it]
 11%|█▏        | 60/522 [24:28<3:01:15, 23.54s/it]
                                                  
+
 11%|█▏        | 60/522 [24:28<3:01:15, 23.54s/it]
 12%|█▏        | 61/522 [24:53<3:03:51, 23.93s/it]
 12%|█▏        | 62/522 [25:14<2:57:30, 23.15s/it]
 12%|█▏        | 63/522 [25:38<2:58:08, 23.29s/it]
 12%|█▏        | 64/522 [26:04<3:03:57, 24.10s/it]
 12%|█▏        | 65/522 [26:22<2:49:52, 22.30s/it]
 13%|█▎        | 66/522 [26:51<3:05:48, 24.45s/it]
 13%|█▎        | 67/522 [27:12<2:56:23, 23.26s/it]
 13%|█▎        | 68/522 [27:37<3:00:37, 23.87s/it]
 13%|█▎        | 69/522 [28:06<3:11:12, 25.32s/it]
 13%|█▎        | 70/522 [28:32<3:12:53, 25.61s/it]
 14%|█▎        | 71/522 [28:59<3:15:50, 26.05s/it]
 14%|█▍        | 72/522 [29:23<3:09:10, 25.22s/it]
 14%|█▍        | 73/522 [29:49<3:10:24, 25.44s/it]
 14%|█▍        | 74/522 [30:15<3:12:14, 25.75s/it]
 14%|█▍        | 75/522 [30:35<2:58:55, 24.02s/it]
 15%|█▍        | 76/522 [30:56<2:52:01, 23.14s/it]
 15%|█▍        | 77/522 [31:22<2:56:50, 23.84s/it]
 15%|█▍        | 78/522 [31:46<2:57:50, 24.03s/it]
 15%|█▌        | 79/522 [32:11<2:58:50, 24.22s/it]
 15%|█▌        | 80/522 [32:32<2:51:56, 23.34s/it]
                                                  
+
 15%|█▌        | 80/522 [32:32<2:51:56, 23.34s/it]
 16%|█▌        | 81/522 [32:53<2:45:32, 22.52s/it]
 16%|█▌        | 82/522 [33:17<2:50:18, 23.22s/it]
 16%|█▌        | 83/522 [33:45<2:58:49, 24.44s/it]
 16%|█▌        | 84/522 [34:11<3:01:57, 24.93s/it]
 16%|█▋        | 85/522 [34:38<3:05:27, 25.46s/it]
 16%|█▋        | 86/522 [35:07<3:14:36, 26.78s/it]
 17%|█▋        | 87/522 [35:30<3:06:12, 25.68s/it]
 17%|█▋        | 88/522 [35:56<3:06:05, 25.73s/it]
 17%|█▋        | 89/522 [36:18<2:56:37, 24.48s/it]
 17%|█▋        | 90/522 [36:46<3:03:31, 25.49s/it]
 17%|█▋        | 91/522 [37:08<2:56:55, 24.63s/it]
 18%|█▊        | 92/522 [37:27<2:44:01, 22.89s/it]
 18%|█▊        | 93/522 [37:53<2:50:29, 23.84s/it]
 18%|█▊        | 94/522 [38:23<3:03:45, 25.76s/it]
 18%|█▊        | 95/522 [38:49<3:02:30, 25.65s/it]
 18%|█▊        | 96/522 [39:13<2:59:25, 25.27s/it]
 19%|█▊        | 97/522 [39:38<2:57:29, 25.06s/it]
 19%|█▉        | 98/522 [40:02<2:55:16, 24.80s/it]
 19%|█▉        | 99/522 [40:25<2:50:50, 24.23s/it]
 19%|█▉        | 100/522 [40:49<2:51:00, 24.31s/it]
                                                   
+
 19%|█▉        | 100/522 [40:49<2:51:00, 24.31s/it]
 19%|█▉        | 101/522 [41:11<2:44:35, 23.46s/it]
 20%|█▉        | 102/522 [41:35<2:46:01, 23.72s/it]
 20%|█▉        | 103/522 [41:59<2:44:48, 23.60s/it]
 20%|█▉        | 104/522 [42:28<2:55:41, 25.22s/it]
 20%|██        | 105/522 [42:51<2:51:57, 24.74s/it]
 20%|██        | 106/522 [43:17<2:54:41, 25.20s/it]
 20%|██        | 107/522 [43:40<2:48:14, 24.32s/it]
 21%|██        | 108/522 [44:00<2:39:14, 23.08s/it]
 21%|██        | 109/522 [44:26<2:44:30, 23.90s/it]
 21%|██        | 110/522 [44:47<2:37:58, 23.01s/it]
 21%|██▏       | 111/522 [45:09<2:36:18, 22.82s/it]
 21%|██▏       | 112/522 [45:28<2:28:14, 21.69s/it]
 22%|██▏       | 113/522 [45:52<2:31:50, 22.28s/it]
 22%|██▏       | 114/522 [46:16<2:34:38, 22.74s/it]
 22%|██▏       | 115/522 [46:38<2:33:14, 22.59s/it]
 22%|██▏       | 116/522 [47:03<2:39:01, 23.50s/it]
 22%|██▏       | 117/522 [47:32<2:49:13, 25.07s/it]
 23%|██▎       | 118/522 [47:56<2:45:57, 24.65s/it]
 23%|██▎       | 119/522 [48:24<2:52:14, 25.64s/it]
 23%|██▎       | 120/522 [48:46<2:45:57, 24.77s/it]
                                                   
+
 23%|██▎       | 120/522 [48:46<2:45:57, 24.77s/it]
 23%|██▎       | 121/522 [49:11<2:45:45, 24.80s/it]
 23%|██▎       | 122/522 [49:40<2:52:32, 25.88s/it]
 24%|██▎       | 123/522 [50:03<2:46:24, 25.02s/it]
 24%|██▍       | 124/522 [50:29<2:48:30, 25.40s/it]
 24%|██▍       | 125/522 [50:52<2:42:25, 24.55s/it]
 24%|██▍       | 126/522 [51:16<2:40:59, 24.39s/it]
 24%|██▍       | 127/522 [51:40<2:39:35, 24.24s/it]
 25%|██▍       | 128/522 [52:04<2:39:20, 24.26s/it]
 25%|██▍       | 129/522 [52:26<2:34:31, 23.59s/it]
 25%|██▍       | 130/522 [52:45<2:25:16, 22.24s/it]
 25%|██▌       | 131/522 [53:11<2:33:09, 23.50s/it]
 25%|██▌       | 132/522 [53:36<2:34:15, 23.73s/it]
 25%|██▌       | 133/522 [53:57<2:29:42, 23.09s/it]
 26%|██▌       | 134/522 [54:25<2:38:58, 24.58s/it]
 26%|██▌       | 135/522 [54:47<2:33:36, 23.81s/it]
 26%|██▌       | 136/522 [55:13<2:36:34, 24.34s/it]
 26%|██▌       | 137/522 [55:34<2:30:29, 23.45s/it]
 26%|█��▋       | 138/522 [56:00<2:33:44, 24.02s/it]
 27%|██▋       | 139/522 [56:24<2:33:06, 23.99s/it]
 27%|██▋       | 140/522 [56:51<2:39:08, 25.00s/it]
                                                   
+
 27%|██▋       | 140/522 [56:51<2:39:08, 25.00s/it]
 27%|██▋       | 141/522 [57:17<2:41:07, 25.37s/it]
 27%|██▋       | 142/522 [57:41<2:38:13, 24.98s/it]
 27%|██▋       | 143/522 [58:06<2:36:32, 24.78s/it]
 28%|██▊       | 144/522 [58:31<2:37:54, 25.07s/it]
 28%|██▊       | 145/522 [58:51<2:28:16, 23.60s/it]
 28%|██▊       | 146/522 [59:14<2:25:07, 23.16s/it]
 28%|██▊       | 147/522 [59:35<2:22:22, 22.78s/it]
 28%|██▊       | 148/522 [1:00:05<2:35:27, 24.94s/it]
 29%|██▊       | 149/522 [1:00:30<2:35:08, 24.96s/it]
 29%|██▊       | 150/522 [1:01:01<2:44:49, 26.59s/it]
 29%|██▉       | 151/522 [1:01:25<2:40:38, 25.98s/it]
 29%|██▉       | 152/522 [1:01:48<2:34:37, 25.07s/it]
 29%|██▉       | 153/522 [1:02:12<2:30:57, 24.55s/it]
 30%|██▉       | 154/522 [1:02:39<2:34:53, 25.25s/it]
 30%|██▉       | 155/522 [1:03:04<2:35:19, 25.39s/it]
 30%|██▉       | 156/522 [1:03:28<2:32:25, 24.99s/it]
 30%|███       | 157/522 [1:03:52<2:29:15, 24.54s/it]
 30%|███       | 158/522 [1:04:12<2:21:32, 23.33s/it]
 30%|███       | 159/522 [1:04:40<2:29:03, 24.64s/it]
 31%|███       | 160/522 [1:05:03<2:25:33, 24.12s/it]
                                                     
+
 31%|███       | 160/522 [1:05:03<2:25:33, 24.12s/it]
 31%|███       | 161/522 [1:05:28<2:26:38, 24.37s/it]
 31%|███       | 162/522 [1:05:50<2:21:48, 23.63s/it]
 31%|███       | 163/522 [1:06:15<2:24:29, 24.15s/it]
 31%|███▏      | 164/522 [1:06:35<2:16:51, 22.94s/it]
 32%|███▏      | 165/522 [1:06:57<2:14:05, 22.54s/it]
 32%|███▏      | 166/522 [1:07:19<2:12:29, 22.33s/it]
 32%|███▏      | 167/522 [1:07:38<2:06:59, 21.46s/it]
 32%|███▏      | 168/522 [1:08:03<2:12:22, 22.44s/it]
 32%|███▏      | 169/522 [1:08:26<2:12:58, 22.60s/it]
 33%|███▎      | 170/522 [1:08:50<2:15:22, 23.07s/it]
 33%|███▎      | 171/522 [1:09:21<2:28:17, 25.35s/it]
 33%|███▎      | 172/522 [1:09:47<2:28:47, 25.51s/it]
 33%|███▎      | 173/522 [1:10:15<2:34:00, 26.48s/it]
 33%|███▎      | 174/522 [1:10:29<2:10:36, 22.52s/it]
 34%|███▎      | 175/522 [1:10:56<2:18:01, 23.87s/it]
 34%|███▎      | 176/522 [1:11:23<2:22:53, 24.78s/it]
 34%|███▍      | 177/522 [1:11:48<2:23:18, 24.92s/it]
 34%|███▍      | 178/522 [1:12:12<2:21:06, 24.61s/it]
 34%|███▍      | 179/522 [1:12:35<2:18:26, 24.22s/it]
 34%|███▍      | 180/522 [1:12:59<2:17:17, 24.09s/it]
                                                     
+
 34%|███▍      | 180/522 [1:12:59<2:17:17, 24.09s/it]
 35%|███▍      | 181/522 [1:13:27<2:24:27, 25.42s/it]
 35%|███▍      | 182/522 [1:13:53<2:24:35, 25.52s/it]
 35%|███▌      | 183/522 [1:14:17<2:21:57, 25.13s/it]
 35%|███▌      | 184/522 [1:14:41<2:19:10, 24.71s/it]
 35%|███▌      | 185/522 [1:15:02<2:12:20, 23.56s/it]
 36%|███▌      | 186/522 [1:15:26<2:12:45, 23.71s/it]
 36%|███▌      | 187/522 [1:15:54<2:20:09, 25.10s/it]
 36%|███▌      | 188/522 [1:16:20<2:21:03, 25.34s/it]
 36%|███▌      | 189/522 [1:16:47<2:22:18, 25.64s/it]
 36%|███▋      | 190/522 [1:17:08<2:14:34, 24.32s/it]
 37%|███▋      | 191/522 [1:17:28<2:07:48, 23.17s/it]
 37%|███▋      | 192/522 [1:17:52<2:08:05, 23.29s/it]
 37%|███▋      | 193/522 [1:18:19<2:13:52, 24.41s/it]
 37%|███▋      | 194/522 [1:18:45<2:16:52, 25.04s/it]
 37%|███▋      | 195/522 [1:19:06<2:10:03, 23.86s/it]
 38%|███▊      | 196/522 [1:19:28<2:06:08, 23.22s/it]
 38%|███▊      | 197/522 [1:19:54<2:09:49, 23.97s/it]
 38%|███▊      | 198/522 [1:20:12<2:00:37, 22.34s/it]
 38%|███▊      | 199/522 [1:20:39<2:06:36, 23.52s/it]
 38%|███▊      | 200/522 [1:21:02<2:06:12, 23.52s/it]
                                                     
+
 38%|███▊      | 200/522 [1:21:02<2:06:12, 23.52s/it]
 39%|███▊      | 201/522 [1:21:31<2:13:59, 25.05s/it]
 39%|███▊      | 202/522 [1:21:56<2:13:03, 24.95s/it]
 39%|███▉      | 203/522 [1:22:19<2:10:13, 24.49s/it]
 39%|███▉      | 204/522 [1:22:46<2:13:58, 25.28s/it]
 39%|███▉      | 205/522 [1:23:10<2:11:47, 24.94s/it]
 39%|███▉      | 206/522 [1:23:33<2:07:41, 24.24s/it]
 40%|███▉      | 207/522 [1:23:58<2:08:43, 24.52s/it]
 40%|███▉      | 208/522 [1:24:21<2:05:13, 23.93s/it]
 40%|████      | 209/522 [1:24:43<2:03:16, 23.63s/it]
 40%|████      | 210/522 [1:25:04<1:57:19, 22.56s/it]
 40%|████      | 211/522 [1:25:30<2:03:26, 23.82s/it]
 41%|████      | 212/522 [1:25:55<2:03:49, 23.96s/it]
 41%|████      | 213/522 [1:26:15<1:57:09, 22.75s/it]
 41%|████      | 214/522 [1:26:36<1:55:07, 22.43s/it]
 41%|████      | 215/522 [1:26:56<1:51:05, 21.71s/it]
 41%|████▏     | 216/522 [1:27:24<2:00:09, 23.56s/it]
 42%|████▏     | 217/522 [1:27:48<2:00:04, 23.62s/it]
 42%|████▏     | 218/522 [1:28:10<1:57:59, 23.29s/it]
 42%|████▏     | 219/522 [1:28:35<2:00:03, 23.77s/it]
 42%|████▏     | 220/522 [1:29:00<2:01:05, 24.06s/it]
                                                     
+
 42%|████▏     | 220/522 [1:29:00<2:01:05, 24.06s/it]
 42%|████▏     | 221/522 [1:29:24<2:00:07, 23.94s/it]
 43%|████▎     | 222/522 [1:29:46<1:57:25, 23.48s/it]
 43%|████▎     | 223/522 [1:30:08<1:54:06, 22.90s/it]
 43%|████▎     | 224/522 [1:30:28<1:49:30, 22.05s/it]
 43%|████▎     | 225/522 [1:30:51<1:51:10, 22.46s/it]
 43%|████▎     | 226/522 [1:31:11<1:46:37, 21.61s/it]
 43%|████▎     | 227/522 [1:31:39<1:55:40, 23.53s/it]
 44%|████▎     | 228/522 [1:32:04<1:57:31, 23.98s/it]
 44%|████▍     | 229/522 [1:32:23<1:50:11, 22.56s/it]
 44%|████▍     | 230/522 [1:32:45<1:48:55, 22.38s/it]
 44%|████▍     | 231/522 [1:33:06<1:45:58, 21.85s/it]
 44%|████▍     | 232/522 [1:33:28<1:46:28, 22.03s/it]
 45%|████▍     | 233/522 [1:33:59<1:58:23, 24.58s/it]
 45%|████▍     | 234/522 [1:34:20<1:52:52, 23.52s/it]
 45%|████▌     | 235/522 [1:34:50<2:02:21, 25.58s/it]
 45%|████▌     | 236/522 [1:35:15<2:01:13, 25.43s/it]
 45%|████▌     | 237/522 [1:35:43<2:04:56, 26.30s/it]
 46%|████▌     | 238/522 [1:36:05<1:57:04, 24.73s/it]
 46%|████▌     | 239/522 [1:36:29<1:56:18, 24.66s/it]
 46%|████▌     | 240/522 [1:36:54<1:56:52, 24.87s/it]
                                                     
+
 46%|████▌     | 240/522 [1:36:54<1:56:52, 24.87s/it]
 46%|████▌     | 241/522 [1:37:19<1:56:11, 24.81s/it]
 46%|████▋     | 242/522 [1:37:49<2:02:40, 26.29s/it]
 47%|████▋     | 243/522 [1:38:13<1:59:32, 25.71s/it]
 47%|████▋     | 244/522 [1:38:35<1:53:33, 24.51s/it]
 47%|████▋     | 245/522 [1:38:57<1:49:48, 23.79s/it]
 47%|████▋     | 246/522 [1:39:20<1:47:58, 23.47s/it]
 47%|████▋     | 247/522 [1:39:47<1:53:31, 24.77s/it]
 48%|████▊     | 248/522 [1:40:10<1:50:33, 24.21s/it]
 48%|████▊     | 249/522 [1:40:33<1:47:51, 23.70s/it]
 48%|████▊     | 250/522 [1:41:00<1:51:27, 24.59s/it]
 48%|████▊     | 251/522 [1:41:23<1:49:47, 24.31s/it]
 48%|████▊     | 252/522 [1:41:46<1:47:53, 23.98s/it]
 48%|████▊     | 253/522 [1:42:13<1:50:21, 24.62s/it]
 49%|████▊     | 254/522 [1:42:40<1:53:17, 25.36s/it]
 49%|████▉     | 255/522 [1:43:07<1:55:47, 26.02s/it]
 49%|████▉     | 256/522 [1:43:28<1:48:30, 24.48s/it]
 49%|████▉     | 257/522 [1:43:54<1:50:41, 25.06s/it]
 49%|████▉     | 258/522 [1:44:22<1:52:58, 25.68s/it]
 50%|████▉     | 259/522 [1:44:43<1:46:55, 24.39s/it]
 50%|████▉     | 260/522 [1:45:04<1:42:14, 23.42s/it]
                                                     
+
 50%|████▉     | 260/522 [1:45:04<1:42:14, 23.42s/it]
 50%|█████     | 261/522 [1:45:25<1:38:31, 22.65s/it]
 50%|█████     | 262/522 [1:45:45<1:35:19, 22.00s/it]
 50%|█████     | 263/522 [1:46:07<1:34:14, 21.83s/it]
 51%|█████     | 264/522 [1:46:29<1:33:46, 21.81s/it]
 51%|█████     | 265/522 [1:46:53<1:36:13, 22.47s/it]
 51%|█████     | 266/522 [1:47:12<1:32:22, 21.65s/it]
 51%|█████     | 267/522 [1:47:31<1:28:28, 20.82s/it]
 51%|█████▏    | 268/522 [1:47:58<1:35:01, 22.45s/it]
 52%|█████▏    | 269/522 [1:48:17<1:31:27, 21.69s/it]
 52%|█████▏    | 270/522 [1:48:42<1:34:55, 22.60s/it]
 52%|█████▏    | 271/522 [1:49:02<1:30:32, 21.64s/it]
 52%|█████▏    | 272/522 [1:49:27<1:34:39, 22.72s/it]
 52%|█████▏    | 273/522 [1:49:49<1:33:59, 22.65s/it]
 52%|█████▏    | 274/522 [1:50:09<1:29:49, 21.73s/it]
 53%|█████▎    | 275/522 [1:50:35<1:34:38, 22.99s/it]
 53%|█████▎    | 276/522 [1:51:00<1:37:10, 23.70s/it]
 53%|█████▎    | 277/522 [1:51:26<1:38:59, 24.24s/it]
 53%|█████▎    | 278/522 [1:51:46<1:33:25, 22.98s/it]
 53%|█████▎    | 279/522 [1:52:12<1:36:44, 23.88s/it]
 54%|���████▎    | 280/522 [1:52:37<1:38:02, 24.31s/it]
                                                     
+
 54%|█████▎    | 280/522 [1:52:37<1:38:02, 24.31s/it]
 54%|█████▍    | 281/522 [1:53:04<1:40:28, 25.02s/it]
 54%|█████▍    | 282/522 [1:53:29<1:40:14, 25.06s/it]
 54%|█████▍    | 283/522 [1:53:51<1:35:54, 24.08s/it]
 54%|█████▍    | 284/522 [1:54:15<1:36:16, 24.27s/it]
 55%|█████▍    | 285/522 [1:54:41<1:37:17, 24.63s/it]
 55%|█████▍    | 286/522 [1:55:04<1:35:09, 24.19s/it]
 55%|█████▍    | 287/522 [1:55:26<1:31:52, 23.46s/it]
 55%|█████▌    | 288/522 [1:55:48<1:30:04, 23.10s/it]
 55%|█████▌    | 289/522 [1:56:13<1:31:42, 23.61s/it]
 56%|█████▌    | 290/522 [1:56:34<1:28:27, 22.88s/it]
 56%|█████▌    | 291/522 [1:56:59<1:30:06, 23.40s/it]
 56%|█████▌    | 292/522 [1:57:21<1:28:27, 23.08s/it]
 56%|█████▌    | 293/522 [1:57:43<1:26:26, 22.65s/it]
 56%|█████▋    | 294/522 [1:58:09<1:30:43, 23.88s/it]
 57%|█████▋    | 295/522 [1:58:31<1:27:56, 23.25s/it]
 57%|█████▋    | 296/522 [1:59:01<1:34:50, 25.18s/it]
 57%|█████▋    | 297/522 [1:59:29<1:37:37, 26.03s/it]
 57%|█████▋    | 298/522 [1:59:57<1:39:32, 26.66s/it]
 57%|█████▋    | 299/522 [2:00:27<1:43:06, 27.74s/it]
 57%|█████▋    | 300/522 [2:00:56<1:43:41, 28.02s/it]
                                                     
+
 57%|█████▋    | 300/522 [2:00:56<1:43:41, 28.02s/it]
 58%|█████▊    | 301/522 [2:01:18<1:36:45, 26.27s/it]
 58%|█████▊    | 302/522 [2:01:47<1:39:10, 27.05s/it]
 58%|█████▊    | 303/522 [2:02:13<1:37:13, 26.64s/it]
 58%|█████▊    | 304/522 [2:02:39<1:36:34, 26.58s/it]
 58%|█████▊    | 305/522 [2:03:04<1:34:17, 26.07s/it]
 59%|█████▊    | 306/522 [2:03:32<1:35:46, 26.60s/it]
 59%|█████▉    | 307/522 [2:03:52<1:28:21, 24.66s/it]
 59%|█████▉    | 308/522 [2:04:19<1:30:50, 25.47s/it]
 59%|█████▉    | 309/522 [2:04:44<1:29:50, 25.31s/it]
 59%|█████▉    | 310/522 [2:05:06<1:25:15, 24.13s/it]
 60%|█████▉    | 311/522 [2:05:29<1:23:57, 23.87s/it]
 60%|█████▉    | 312/522 [2:05:54<1:25:04, 24.31s/it]
 60%|█████▉    | 313/522 [2:06:25<1:31:00, 26.13s/it]
 60%|██████    | 314/522 [2:06:47<1:26:41, 25.00s/it]
 60%|██████    | 315/522 [2:07:09<1:23:23, 24.17s/it]
 61%|██████    | 316/522 [2:07:40<1:29:33, 26.08s/it]
 61%|██████    | 317/522 [2:08:06<1:28:54, 26.02s/it]
 61%|██████    | 318/522 [2:08:30<1:26:46, 25.52s/it]
 61%|██████    | 319/522 [2:08:58<1:28:42, 26.22s/it]
 61%|██████▏   | 320/522 [2:09:24<1:28:28, 26.28s/it]
                                                     
+
 61%|██████▏   | 320/522 [2:09:24<1:28:28, 26.28s/it]
 61%|██████▏   | 321/522 [2:09:44<1:21:03, 24.19s/it]
 62%|██████▏   | 322/522 [2:10:09<1:21:57, 24.59s/it]
 62%|██████▏   | 323/522 [2:10:34<1:22:11, 24.78s/it]
 62%|██████▏   | 324/522 [2:10:54<1:16:56, 23.31s/it]
 62%|██████▏   | 325/522 [2:11:22<1:21:10, 24.72s/it]
 62%|██████▏   | 326/522 [2:11:48<1:22:13, 25.17s/it]
 63%|██████▎   | 327/522 [2:12:18<1:26:12, 26.52s/it]
 63%|██████▎   | 328/522 [2:12:43<1:23:46, 25.91s/it]
 63%|██████▎   | 329/522 [2:13:09<1:23:43, 26.03s/it]
 63%|██████▎   | 330/522 [2:13:34<1:22:09, 25.68s/it]
 63%|██████▎   | 331/522 [2:13:54<1:16:22, 23.99s/it]
 64%|██████▎   | 332/522 [2:14:23<1:20:30, 25.42s/it]
 64%|██████▍   | 333/522 [2:14:46<1:18:16, 24.85s/it]
 64%|██████▍   | 334/522 [2:15:17<1:23:15, 26.57s/it]
 64%|██████▍   | 335/522 [2:15:39<1:18:51, 25.30s/it]
 64%|██████▍   | 336/522 [2:16:02<1:15:54, 24.49s/it]
 65%|██████▍   | 337/522 [2:16:30<1:19:24, 25.76s/it]
 65%|██████▍   | 338/522 [2:16:53<1:16:31, 24.95s/it]
 65%|██████▍   | 339/522 [2:17:19<1:16:53, 25.21s/it]
 65%|██████▌   | 340/522 [2:17:44<1:16:25, 25.20s/it]
                                                     
+
 65%|██████▌   | 340/522 [2:17:44<1:16:25, 25.20s/it]
 65%|██████▌   | 341/522 [2:18:12<1:18:15, 25.94s/it]
 66%|██████▌   | 342/522 [2:18:37<1:16:47, 25.60s/it]
 66%|██████▌   | 343/522 [2:19:03<1:16:49, 25.75s/it]
 66%|██████▌   | 344/522 [2:19:24<1:12:40, 24.50s/it]
 66%|██████▌   | 345/522 [2:19:45<1:08:54, 23.36s/it]
 66%|██████▋   | 346/522 [2:20:11<1:11:06, 24.24s/it]
 66%|██████▋   | 347/522 [2:20:32<1:07:16, 23.07s/it]
 67%|██████▋   | 348/522 [2:20:46<58:45, 20.26s/it]  
 67%|██████▋   | 349/522 [2:21:12<1:04:03, 22.21s/it]
 67%|██████▋   | 350/522 [2:21:42<1:09:44, 24.33s/it]
 67%|██████▋   | 351/522 [2:22:08<1:11:11, 24.98s/it]
 67%|██████▋   | 352/522 [2:22:34<1:11:46, 25.33s/it]
 68%|██████▊   | 353/522 [2:23:01<1:12:27, 25.72s/it]
 68%|██████▊   | 354/522 [2:23:23<1:08:52, 24.60s/it]
 68%|██████▊   | 355/522 [2:23:48<1:08:32, 24.63s/it]
 68%|██████▊   | 356/522 [2:24:11<1:07:30, 24.40s/it]
 68%|██████▊   | 357/522 [2:24:30<1:02:15, 22.64s/it]
 69%|██████▊   | 358/522 [2:24:50<59:27, 21.75s/it]  
 69%|██████▉   | 359/522 [2:25:17<1:03:36, 23.42s/it]
 69%|██████▉   | 360/522 [2:25:39<1:02:29, 23.14s/it]
                                                     
+
 69%|██████▉   | 360/522 [2:25:39<1:02:29, 23.14s/it]
 69%|██████▉   | 361/522 [2:26:01<1:00:30, 22.55s/it]
 69%|██████▉   | 362/522 [2:26:24<1:00:30, 22.69s/it]
 70%|██████▉   | 363/522 [2:26:54<1:06:09, 24.96s/it]
 70%|██████▉   | 364/522 [2:27:15<1:02:23, 23.70s/it]
 70%|██████▉   | 365/522 [2:27:39<1:02:19, 23.82s/it]
 70%|███████   | 366/522 [2:28:00<59:49, 23.01s/it]  
 70%|███████   | 367/522 [2:28:26<1:02:09, 24.06s/it]
 70%|███████   | 368/522 [2:28:49<1:00:38, 23.63s/it]
 71%|███████   | 369/522 [2:29:16<1:02:34, 24.54s/it]
 71%|███████   | 370/522 [2:29:40<1:02:23, 24.63s/it]
 71%|███████   | 371/522 [2:30:07<1:03:15, 25.14s/it]
 71%|███████▏  | 372/522 [2:30:33<1:03:43, 25.49s/it]
 71%|███████▏  | 373/522 [2:30:57<1:02:09, 25.03s/it]
 72%|███████▏  | 374/522 [2:31:23<1:02:25, 25.31s/it]
 72%|███████▏  | 375/522 [2:31:46<1:00:15, 24.60s/it]
 72%|███████▏  | 376/522 [2:32:12<1:00:57, 25.05s/it]
 72%|███████▏  | 377/522 [2:32:33<57:32, 23.81s/it]  
 72%|███████▏  | 378/522 [2:33:00<59:10, 24.66s/it]
 73%|███████▎  | 379/522 [2:33:20<55:51, 23.44s/it]
 73%|███████▎  | 380/522 [2:33:46<57:29, 24.29s/it]
                                                   
+
 73%|███████▎  | 380/522 [2:33:46<57:29, 24.29s/it]
 73%|███████▎  | 381/522 [2:34:11<56:56, 24.23s/it]
 73%|███████▎  | 382/522 [2:34:41<1:00:42, 26.02s/it]
 73%|███████▎  | 383/522 [2:35:10<1:02:42, 27.07s/it]
 74%|███████▎  | 384/522 [2:35:38<1:02:39, 27.24s/it]
 74%|███████▍  | 385/522 [2:36:02<1:00:05, 26.32s/it]
 74%|███████▍  | 386/522 [2:36:29<59:47, 26.38s/it]  
 74%|███████▍  | 387/522 [2:36:54<58:49, 26.14s/it]
 74%|███████▍  | 388/522 [2:37:21<58:47, 26.32s/it]
 75%|███████▍  | 389/522 [2:37:42<54:47, 24.72s/it]
 75%|███████▍  | 390/522 [2:38:03<51:44, 23.52s/it]
 75%|███████▍  | 391/522 [2:38:25<50:31, 23.14s/it]
 75%|███████▌  | 392/522 [2:38:50<51:43, 23.87s/it]
 75%|███████▌  | 393/522 [2:39:20<55:05, 25.63s/it]
 75%|███████▌  | 394/522 [2:39:41<51:40, 24.22s/it]
 76%|███████▌  | 395/522 [2:40:03<49:58, 23.61s/it]
 76%|███████▌  | 396/522 [2:40:29<50:37, 24.11s/it]
 76%|███████▌  | 397/522 [2:40:49<48:10, 23.13s/it]
 76%|███████▌  | 398/522 [2:41:14<48:26, 23.44s/it]
 76%|███████▋  | 399/522 [2:41:41<50:35, 24.67s/it]
 77%|███████▋  | 400/522 [2:42:06<50:13, 24.70s/it]
                                                   
+
 77%|███████▋  | 400/522 [2:42:06<50:13, 24.70s/it]
 77%|███████▋  | 401/522 [2:42:29<49:04, 24.34s/it]
 77%|███████▋  | 402/522 [2:42:59<52:07, 26.06s/it]
 77%|███████▋  | 403/522 [2:43:27<52:20, 26.39s/it]
 77%|███████▋  | 404/522 [2:43:52<51:31, 26.20s/it]
 78%|███████▊  | 405/522 [2:44:14<48:27, 24.85s/it]
 78%|███████▊  | 406/522 [2:44:40<48:41, 25.18s/it]
 78%|███████▊  | 407/522 [2:45:03<46:50, 24.44s/it]
 78%|███████▊  | 408/522 [2:45:24<44:23, 23.37s/it]
 78%|███████▊  | 409/522 [2:45:47<44:09, 23.45s/it]
 79%|███████▊  | 410/522 [2:46:08<42:09, 22.58s/it]
 79%|███████▊  | 411/522 [2:46:30<41:29, 22.43s/it]
 79%|███████▉  | 412/522 [2:46:55<42:38, 23.26s/it]
 79%|███████▉  | 413/522 [2:47:18<41:56, 23.09s/it]
 79%|███████▉  | 414/522 [2:47:45<43:59, 24.44s/it]
 80%|███████▉  | 415/522 [2:48:05<40:55, 22.95s/it]
 80%|███████▉  | 416/522 [2:48:29<40:55, 23.17s/it]
 80%|███████▉  | 417/522 [2:48:53<41:10, 23.53s/it]
 80%|████████  | 418/522 [2:49:15<39:53, 23.02s/it]
 80%|████████  | 419/522 [2:49:42<41:56, 24.43s/it]
 80%|████████  | 420/522 [2:50:08<42:07, 24.78s/it]
                                                   
+
 80%|████████  | 420/522 [2:50:08<42:07, 24.78s/it]
 81%|████████  | 421/522 [2:50:30<40:06, 23.83s/it]
 81%|████████  | 422/522 [2:50:54<39:57, 23.98s/it]
 81%|████████  | 423/522 [2:51:15<38:13, 23.16s/it]
 81%|████████  | 424/522 [2:51:43<39:55, 24.44s/it]
 81%|████████▏ | 425/522 [2:52:06<38:55, 24.08s/it]
 82%|████████▏ | 426/522 [2:52:28<37:38, 23.53s/it]
 82%|████████▏ | 427/522 [2:52:51<37:05, 23.43s/it]
 82%|████████▏ | 428/522 [2:53:13<35:55, 22.93s/it]
 82%|████████▏ | 429/522 [2:53:41<37:50, 24.42s/it]
 82%|████████▏ | 430/522 [2:54:03<36:31, 23.83s/it]
 83%|████████▎ | 431/522 [2:54:29<36:55, 24.34s/it]
 83%|████████▎ | 432/522 [2:54:50<34:54, 23.28s/it]
 83%|████████▎ | 433/522 [2:55:15<35:30, 23.94s/it]
 83%|████████▎ | 434/522 [2:55:41<35:50, 24.44s/it]
 83%|████████▎ | 435/522 [2:56:10<37:18, 25.73s/it]
 84%|████████▎ | 436/522 [2:56:33<35:58, 25.10s/it]
 84%|████████▎ | 437/522 [2:56:58<35:30, 25.06s/it]
 84%|████████▍ | 438/522 [2:57:21<34:08, 24.39s/it]
 84%|████████▍ | 439/522 [2:57:42<32:14, 23.30s/it]
 84%|████████▍ | 440/522 [2:58:04<31:31, 23.07s/it]
                                                   
+
 84%|████████▍ | 440/522 [2:58:04<31:31, 23.07s/it]
 84%|████████▍ | 441/522 [2:58:29<31:44, 23.51s/it]
 85%|████████▍ | 442/522 [2:58:55<32:25, 24.32s/it]
 85%|████████▍ | 443/522 [2:59:21<32:49, 24.93s/it]
 85%|████████▌ | 444/522 [2:59:49<33:27, 25.74s/it]
 85%|████████▌ | 445/522 [3:00:15<32:58, 25.69s/it]
 85%|████████▌ | 446/522 [3:00:36<30:49, 24.34s/it]
 86%|████████▌ | 447/522 [3:00:59<30:00, 24.01s/it]
 86%|████████▌ | 448/522 [3:01:28<31:16, 25.36s/it]
 86%|████████▌ | 449/522 [3:01:49<29:15, 24.05s/it]
 86%|████████▌ | 450/522 [3:02:08<27:04, 22.57s/it]
 86%|████████▋ | 451/522 [3:02:37<29:09, 24.64s/it]
 87%|████████▋ | 452/522 [3:02:59<27:44, 23.78s/it]
 87%|████████▋ | 453/522 [3:03:18<25:49, 22.46s/it]
 87%|████████▋ | 454/522 [3:03:44<26:24, 23.30s/it]
 87%|████████▋ | 455/522 [3:04:12<27:43, 24.82s/it]
 87%|████████▋ | 456/522 [3:04:33<26:01, 23.66s/it]
 88%|████████▊ | 457/522 [3:05:01<27:06, 25.02s/it]
 88%|████████▊ | 458/522 [3:05:28<27:26, 25.73s/it]
 88%|████████▊ | 459/522 [3:05:55<27:17, 25.99s/it]
 88%|████████▊ | 460/522 [3:06:20<26:36, 25.75s/it]
                                                   
+
 88%|████████▊ | 460/522 [3:06:20<26:36, 25.75s/it]
 88%|████████▊ | 461/522 [3:06:46<26:02, 25.62s/it]
 89%|████████▊ | 462/522 [3:07:10<25:18, 25.30s/it]
 89%|████████▊ | 463/522 [3:07:31<23:27, 23.85s/it]
 89%|████████▉ | 464/522 [3:07:49<21:30, 22.24s/it]
 89%|████████▉ | 465/522 [3:08:16<22:28, 23.65s/it]
 89%|████████▉ | 466/522 [3:08:37<21:20, 22.86s/it]
 89%|████████▉ | 467/522 [3:09:04<22:00, 24.01s/it]
 90%|████████▉ | 468/522 [3:09:31<22:36, 25.12s/it]
 90%|████████▉ | 469/522 [3:09:58<22:30, 25.49s/it]
 90%|█████████ | 470/522 [3:10:19<20:59, 24.21s/it]
 90%|█████████ | 471/522 [3:10:46<21:16, 25.04s/it]
 90%|█████████ | 472/522 [3:11:09<20:20, 24.41s/it]
 91%|█████████ | 473/522 [3:11:32<19:31, 23.90s/it]
 91%|█████████ | 474/522 [3:11:52<18:18, 22.88s/it]
 91%|█████████ | 475/522 [3:12:20<19:06, 24.39s/it]
 91%|█████████ | 476/522 [3:12:51<20:06, 26.23s/it]
 91%|█████████▏| 477/522 [3:13:13<18:55, 25.23s/it]
 92%|█████████▏| 478/522 [3:13:40<18:41, 25.49s/it]
 92%|█████████▏| 479/522 [3:14:03<17:49, 24.88s/it]
 92%|█████████▏| 480/522 [3:14:28<17:23, 24.85s/it]
                                                   
+
 92%|█████████▏| 480/522 [3:14:28<17:23, 24.85s/it]
 92%|█████████▏| 481/522 [3:14:47<15:47, 23.11s/it]
 92%|█████████▏| 482/522 [3:15:13<15:57, 23.94s/it]
 93%|█████████▎| 483/522 [3:15:37<15:36, 24.02s/it]
 93%|█████████▎| 484/522 [3:16:00<15:02, 23.75s/it]
 93%|█████████▎| 485/522 [3:16:21<14:10, 23.00s/it]
 93%|█████████▎| 486/522 [3:16:47<14:12, 23.69s/it]
 93%|█████████▎| 487/522 [3:17:08<13:27, 23.06s/it]
 93%|█████████▎| 488/522 [3:17:29<12:43, 22.46s/it]
 94%|█████████▎| 489/522 [3:17:52<12:27, 22.65s/it]
 94%|█████████▍| 490/522 [3:18:13<11:45, 22.06s/it]
 94%|█████████▍| 491/522 [3:18:37<11:38, 22.54s/it]
 94%|█████████▍| 492/522 [3:18:58<11:06, 22.21s/it]
 94%|█████████▍| 493/522 [3:19:21<10:49, 22.41s/it]
 95%|█████████▍| 494/522 [3:19:46<10:51, 23.26s/it]
 95%|█████████▍| 495/522 [3:20:12<10:47, 23.99s/it]
 95%|█████████▌| 496/522 [3:20:41<11:05, 25.59s/it]
 95%|█████████▌| 497/522 [3:21:04<10:21, 24.87s/it]
 95%|█████████▌| 498/522 [3:21:30<09:59, 24.99s/it]
 96%|█████████▌| 499/522 [3:21:55<09:33, 24.93s/it]
 96%|█████████▌| 500/522 [3:22:18<08:58, 24.47s/it]
                                                   
+
 96%|█████████▌| 500/522 [3:22:18<08:58, 24.47s/it]
 96%|█████████▌| 501/522 [3:22:47<09:01, 25.80s/it]
 96%|█████████▌| 502/522 [3:23:17<09:01, 27.09s/it]
 96%|█████████▋| 503/522 [3:23:38<08:01, 25.35s/it]
 97%|█████████▋| 504/522 [3:24:07<07:55, 26.41s/it]
 97%|█████████▋| 505/522 [3:24:32<07:19, 25.84s/it]
 97%|█████████▋| 506/522 [3:24:52<06:24, 24.05s/it]
 97%|█████████▋| 507/522 [3:25:18<06:10, 24.68s/it]
 97%|█████████▋| 508/522 [3:25:41<05:40, 24.33s/it]
 98%|█████████▊| 509/522 [3:26:01<05:00, 23.08s/it]
 98%|█████████▊| 510/522 [3:26:31<05:00, 25.08s/it]
 98%|█████████▊| 511/522 [3:26:58<04:42, 25.71s/it]
 98%|█████████▊| 512/522 [3:27:21<04:08, 24.84s/it]
 98%|█████████▊| 513/522 [3:27:51<03:58, 26.51s/it]
 98%|█████████▊| 514/522 [3:28:14<03:21, 25.23s/it]
 99%|█████████▊| 515/522 [3:28:36<02:50, 24.41s/it]
 99%|█████████▉| 516/522 [3:28:59<02:24, 24.04s/it]
 99%|█████████▉| 517/522 [3:29:22<01:58, 23.72s/it]
 99%|█████████▉| 518/522 [3:29:42<01:29, 22.37s/it]
 99%|█████████▉| 519/522 [3:30:02<01:05, 21.92s/it]
                                                   
+
                                                   
+
+***** train metrics *****
+  epoch                    =        3.0
+  train_loss               =     0.9344
+  train_runtime            = 3:31:10.72
+  train_samples_per_second =      0.822
+  train_steps_per_second   =      0.041

without_ncr_ft/train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.9344269825124193,
+    "train_runtime": 12670.7287,
+    "train_samples_per_second": 0.822,
+    "train_steps_per_second": 0.041
+}

without_ncr_ft/trainer_state.json

@@ -0,0 +1,210 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 522,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.11,
+      "global_step": 20,
+      "learning_rate": 3.39622641509434e-05,
+      "loss": 5.6443,
+      "step": 20
+    },
+    {
+      "epoch": 0.23,
+      "global_step": 40,
+      "learning_rate": 7.169811320754717e-05,
+      "loss": 1.5982,
+      "step": 40
+    },
+    {
+      "epoch": 0.34,
+      "global_step": 60,
+      "learning_rate": 0.0001,
+      "loss": 0.7384,
+      "step": 60
+    },
+    {
+      "epoch": 0.46,
+      "global_step": 80,
+      "learning_rate": 0.0001,
+      "loss": 0.7772,
+      "step": 80
+    },
+    {
+      "epoch": 0.57,
+      "global_step": 100,
+      "learning_rate": 0.0001,
+      "loss": 0.7537,
+      "step": 100
+    },
+    {
+      "epoch": 0.69,
+      "global_step": 120,
+      "learning_rate": 0.0001,
+      "loss": 0.7405,
+      "step": 120
+    },
+    {
+      "epoch": 0.8,
+      "global_step": 140,
+      "learning_rate": 0.0001,
+      "loss": 0.716,
+      "step": 140
+    },
+    {
+      "epoch": 0.92,
+      "global_step": 160,
+      "learning_rate": 0.0001,
+      "loss": 0.7134,
+      "step": 160
+    },
+    {
+      "epoch": 1.03,
+      "global_step": 180,
+      "learning_rate": 0.0001,
+      "loss": 0.7104,
+      "step": 180
+    },
+    {
+      "epoch": 1.15,
+      "global_step": 200,
+      "learning_rate": 0.0001,
+      "loss": 0.721,
+      "step": 200
+    },
+    {
+      "epoch": 1.26,
+      "global_step": 220,
+      "learning_rate": 0.0001,
+      "loss": 0.7116,
+      "step": 220
+    },
+    {
+      "epoch": 1.38,
+      "global_step": 240,
+      "learning_rate": 0.0001,
+      "loss": 0.7153,
+      "step": 240
+    },
+    {
+      "epoch": 1.49,
+      "global_step": 260,
+      "learning_rate": 0.0001,
+      "loss": 0.7116,
+      "step": 260
+    },
+    {
+      "epoch": 1.61,
+      "global_step": 280,
+      "learning_rate": 0.0001,
+      "loss": 0.6967,
+      "step": 280
+    },
+    {
+      "epoch": 1.72,
+      "global_step": 300,
+      "learning_rate": 0.0001,
+      "loss": 0.7469,
+      "step": 300
+    },
+    {
+      "epoch": 1.84,
+      "global_step": 320,
+      "learning_rate": 0.0001,
+      "loss": 0.6993,
+      "step": 320
+    },
+    {
+      "epoch": 1.95,
+      "global_step": 340,
+      "learning_rate": 0.0001,
+      "loss": 0.7135,
+      "step": 340
+    },
+    {
+      "epoch": 2.07,
+      "global_step": 360,
+      "learning_rate": 0.0001,
+      "loss": 0.682,
+      "step": 360
+    },
+    {
+      "epoch": 2.18,
+      "global_step": 380,
+      "learning_rate": 0.0001,
+      "loss": 0.6944,
+      "step": 380
+    },
+    {
+      "epoch": 2.3,
+      "global_step": 400,
+      "learning_rate": 0.0001,
+      "loss": 0.6939,
+      "step": 400
+    },
+    {
+      "epoch": 2.41,
+      "global_step": 420,
+      "learning_rate": 0.0001,
+      "loss": 0.6787,
+      "step": 420
+    },
+    {
+      "epoch": 2.53,
+      "global_step": 440,
+      "learning_rate": 0.0001,
+      "loss": 0.6928,
+      "step": 440
+    },
+    {
+      "epoch": 2.64,
+      "global_step": 460,
+      "learning_rate": 0.0001,
+      "loss": 0.7083,
+      "step": 460
+    },
+    {
+      "epoch": 2.76,
+      "global_step": 480,
+      "learning_rate": 0.0001,
+      "loss": 0.694,
+      "step": 480
+    },
+    {
+      "epoch": 2.87,
+      "global_step": 500,
+      "learning_rate": 0.0001,
+      "loss": 0.688,
+      "step": 500
+    },
+    {
+      "epoch": 2.99,
+      "global_step": 520,
+      "learning_rate": 0.0001,
+      "loss": 0.6821,
+      "step": 520
+    },
+    {
+      "epoch": 3.0,
+      "step": 522,
+      "total_flos": 2.7659858036444365e+17,
+      "train_loss": 0.9344269825124193,
+      "train_runtime": 12670.7287,
+      "train_samples_per_second": 0.822,
+      "train_steps_per_second": 0.041
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 522,
+  "num_train_epochs": 3,
+  "save_steps": 200,
+  "total_flos": 2.7659858036444365e+17,
+  "trial_name": null,
+  "trial_params": null
+}

without_race_ft/all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 1.2810182730356852,
+    "train_runtime": 12278.7402,
+    "train_samples_per_second": 0.411,
+    "train_steps_per_second": 0.034
+}

without_race_ft/final/README.md

@@ -0,0 +1,21 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: float16
+### Framework versions
+
+
+- PEFT 0.5.0

without_race_ft/final/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "/data0/maqi/huggingface_models/llama-2-7b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

without_race_ft/final/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f7ace4f52e9a4a84153eaa506dbdea489af7cf05913352ea58942834710ad8
+size 1279424269

without_race_ft/final/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92a4c148384794396a3af608d1ed05a9716553724dad573651248833fdff3195
+size 4091

without_race_ft/train.log

@@ -0,0 +1,32 @@
+model training desc:  不在race数据集上预训练,仅使用quality数据集训练
+2023-12-05 13:07:05.137 | INFO     | __main__:init_components:108 - Initializing components...
+
+You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. If you see this, DO NOT PANIC! This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thouroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
+2023-12-05 13:07:59.679 | INFO     | __main__:init_components:155 - 
+
+2023-12-05 13:07:59.679 | INFO     | __main__:init_components:156 - ********************
+2023-12-05 13:07:59.679 | INFO     | __main__:init_components:157 - using llama2 model
+2023-12-05 13:07:59.680 | INFO     | __main__:init_components:158 - ********************
+2023-12-05 13:07:59.680 | INFO     | __main__:init_components:159 - 
+
+memory footprint of model: 4.024436950683594 GB
+trainable params: 319,815,680 || all params: 7,058,231,296 || trainable%: 4.531102291607305
+2023-12-05 13:08:41.045 | INFO     | component.dataset:__init__:14 - Loading data: /data0/maqi/KGLQA-data/datasets/QuALITY/quality_rocketqa_2048_instruct/train.jsonl
+2023-12-05 13:08:41.079 | INFO     | component.dataset:__init__:19 - there are 2523 data in dataset
+2023-12-05 13:08:41.216 | INFO     | __main__:main:231 - *** starting training ***
+
  0%|          | 0/420 [00:00<?, ?it/s]
  0%|          | 1/420 [00:34<4:00:58, 34.51s/it]
  0%|          | 2/420 [01:04<3:40:50, 31.70s/it]
  1%|          | 3/420 [01:33<3:32:18, 30.55s/it]
  1%|          | 4/420 [02:02<3:28:07, 30.02s/it]
  1%|          | 5/420 [02:31<3:24:50, 29.62s/it]
  1%|▏         | 6/420 [03:00<3:23:28, 29.49s/it]
  2%|▏         | 7/420 [03:30<3:23:12, 29.52s/it]
  2%|▏         | 8/420 [03:59<3:21:25, 29.33s/it]
  2%|▏         | 9/420 [04:28<3:20:05, 29.21s/it]
  2%|▏         | 10/420 [04:57<3:19:46, 29.23s/it]
  3%|▎         | 11/420 [05:26<3:19:05, 29.21s/it]
  3%|▎         | 12/420 [05:55<3:18:23, 29.18s/it]
  3%|▎         | 13/420 [06:25<3:18:44, 29.30s/it]
  3%|▎         | 14/420 [06:54<3:18:11, 29.29s/it]
  4%|▎         | 15/420 [07:23<3:17:43, 29.29s/it]
  4%|▍         | 16/420 [07:52<3:16:23, 29.17s/it]
  4%|▍         | 17/420 [08:22<3:16:05, 29.19s/it]
  4%|▍         | 18/420 [08:51<3:16:23, 29.31s/it]
  5%|▍         | 19/420 [09:20<3:15:48, 29.30s/it]
  5%|▍         | 20/420 [09:49<3:14:55, 29.24s/it]
  5%|▌         | 21/420 [10:19<3:14:27, 29.24s/it]
  5%|▌         | 22/420 [10:48<3:13:58, 29.24s/it]
  5%|▌         | 23/420 [11:18<3:14:09, 29.34s/it]
  6%|▌         | 24/420 [11:47<3:12:52, 29.22s/it]
  6%|▌         | 25/420 [12:16<3:12:12, 29.20s/it]
  6%|▌         | 26/420 [12:45<3:11:45, 29.20s/it]
  6%|▋         | 27/420 [13:14<3:11:19, 29.21s/it]
  7%|▋         | 28/420 [13:43<3:10:18, 29.13s/it]
  7%|▋         | 29/420 [14:12<3:10:20, 29.21s/it]
  7%|▋         | 30/420 [14:42<3:10:36, 29.32s/it]
  7%|▋         | 31/420 [15:11<3:09:21, 29.21s/it]
  8%|▊         | 32/420 [15:41<3:09:34, 29.31s/it]
  8%|▊         | 33/420 [16:10<3:08:57, 29.30s/it]
  8%|▊         | 34/420 [16:39<3:07:28, 29.14s/it]
  8%|▊         | 35/420 [17:08<3:07:13, 29.18s/it]
  9%|▊         | 36/420 [17:37<3:06:51, 29.20s/it]
  9%|▉         | 37/420 [18:06<3:05:49, 29.11s/it]
  9%|▉         | 38/420 [18:36<3:07:11, 29.40s/it]
  9%|▉         | 39/420 [19:05<3:06:27, 29.36s/it]
 10%|▉         | 40/420 [19:34<3:05:04, 29.22s/it]
 10%|▉         | 41/420 [20:03<3:04:39, 29.23s/it]
 10%|█         | 42/420 [20:33<3:03:55, 29.19s/it]
 10%|█         | 43/420 [21:02<3:03:31, 29.21s/it]
 10%|█         | 44/420 [21:31<3:03:41, 29.31s/it]
 11%|█         | 45/420 [22:01<3:03:01, 29.28s/it]
 11%|█         | 46/420 [22:30<3:02:58, 29.36s/it]
 11%|█         | 47/420 [22:59<3:01:51, 29.25s/it]
 11%|█▏        | 48/420 [23:28<3:01:15, 29.23s/it]
 12%|█▏        | 49/420 [23:58<3:01:18, 29.32s/it]
 12%|█▏        | 50/420 [24:27<3:00:35, 29.29s/it]
                                                  
+
 12%|█▏        | 50/420 [24:27<3:00:35, 29.29s/it]
 12%|█▏        | 51/420 [24:56<2:59:26, 29.18s/it]
 12%|█▏        | 52/420 [25:25<2:58:31, 29.11s/it]
 13%|█▎        | 53/420 [25:54<2:58:15, 29.14s/it]
 13%|█▎        | 54/420 [26:23<2:57:54, 29.17s/it]
 13%|█▎        | 55/420 [26:53<2:57:31, 29.18s/it]
 13%|█▎        | 56/420 [27:22<2:57:43, 29.30s/it]
 14%|█▎        | 57/420 [27:51<2:56:31, 29.18s/it]
 14%|█▍        | 58/420 [28:20<2:56:09, 29.20s/it]
 14%|█▍        | 59/420 [28:50<2:55:43, 29.21s/it]
 14%|█▍        | 60/420 [29:18<2:54:46, 29.13s/it]
 15%|█▍        | 61/420 [29:48<2:55:00, 29.25s/it]
 15%|█▍        | 62/420 [30:17<2:54:28, 29.24s/it]
 15%|█▌        | 63/420 [30:47<2:54:30, 29.33s/it]
 15%|█▌        | 64/420 [31:16<2:53:14, 29.20s/it]
 15%|█▌        | 65/420 [31:45<2:52:16, 29.12s/it]
 16%|█▌        | 66/420 [32:14<2:51:58, 29.15s/it]
 16%|█▌        | 67/420 [32:43<2:51:03, 29.08s/it]
 16%|█▌        | 68/420 [33:12<2:50:11, 29.01s/it]
 16%|█▋        | 69/420 [33:41<2:50:39, 29.17s/it]
 17%|█▋        | 70/420 [34:10<2:49:40, 29.09s/it]
 17%|█▋        | 71/420 [34:40<2:50:01, 29.23s/it]
 17%|█▋        | 72/420 [35:08<2:48:30, 29.05s/it]
 17%|█▋        | 73/420 [35:38<2:48:52, 29.20s/it]
 18%|█▊        | 74/420 [36:07<2:48:11, 29.17s/it]
 18%|█▊        | 75/420 [36:36<2:47:47, 29.18s/it]
 18%|█▊        | 76/420 [37:06<2:47:55, 29.29s/it]
 18%|█▊        | 77/420 [37:35<2:47:54, 29.37s/it]
 19%|█▊        | 78/420 [38:04<2:46:19, 29.18s/it]
 19%|█▉        | 79/420 [38:33<2:46:28, 29.29s/it]
 19%|█▉        | 80/420 [39:02<2:45:03, 29.13s/it]
 19%|█▉        | 81/420 [39:31<2:44:43, 29.15s/it]
 20%|█▉        | 82/420 [40:01<2:44:21, 29.18s/it]
 20%|█▉        | 83/420 [40:30<2:43:52, 29.18s/it]
 20%|██        | 84/420 [40:59<2:42:52, 29.08s/it]
 20%|██        | 85/420 [41:28<2:43:08, 29.22s/it]
 20%|██        | 86/420 [41:57<2:42:27, 29.18s/it]
 21%|██        | 87/420 [42:26<2:41:29, 29.10s/it]
 21%|██        | 88/420 [42:55<2:40:59, 29.10s/it]
 21%|██        | 89/420 [43:25<2:40:43, 29.13s/it]
 21%|██▏       | 90/420 [43:54<2:40:24, 29.16s/it]
 22%|██▏       | 91/420 [44:23<2:39:43, 29.13s/it]
 22%|██▏       | 92/420 [44:52<2:39:25, 29.16s/it]
 22%|██▏       | 93/420 [45:21<2:39:02, 29.18s/it]
 22%|██▏       | 94/420 [45:50<2:38:31, 29.18s/it]
 23%|██▎       | 95/420 [46:20<2:38:04, 29.18s/it]
 23%|██▎       | 96/420 [46:49<2:37:24, 29.15s/it]
 23%|██▎       | 97/420 [47:18<2:37:01, 29.17s/it]
 23%|██▎       | 98/420 [47:47<2:37:11, 29.29s/it]
 24%|██▎       | 99/420 [48:17<2:37:07, 29.37s/it]
 24%|██▍       | 100/420 [48:46<2:36:24, 29.33s/it]
                                                   
+
 24%|██▍       | 100/420 [48:46<2:36:24, 29.33s/it]
 24%|██▍       | 101/420 [49:15<2:35:32, 29.26s/it]
 24%|██▍       | 102/420 [49:45<2:35:00, 29.25s/it]
 25%|██▍       | 103/420 [50:14<2:34:27, 29.23s/it]
 25%|██▍       | 104/420 [50:43<2:33:53, 29.22s/it]
 25%|██▌       | 105/420 [51:12<2:33:52, 29.31s/it]
 25%|██▌       | 106/420 [51:42<2:33:41, 29.37s/it]
 25%|██▌       | 107/420 [52:11<2:33:00, 29.33s/it]
 26%|██▌       | 108/420 [52:40<2:32:08, 29.26s/it]
 26%|██▌       | 109/420 [53:09<2:31:06, 29.15s/it]
 26%|██▌       | 110/420 [53:38<2:30:41, 29.17s/it]
 26%|██▋       | 111/420 [54:07<2:29:48, 29.09s/it]
 27%|██▋       | 112/420 [54:37<2:29:31, 29.13s/it]
 27%|██▋       | 113/420 [55:06<2:29:42, 29.26s/it]
 27%|██▋       | 114/420 [55:35<2:28:53, 29.20s/it]
 27%|██▋       | 115/420 [56:04<2:28:17, 29.17s/it]
 28%|██▊       | 116/420 [56:34<2:27:52, 29.19s/it]
 28%|██▊       | 117/420 [57:02<2:27:00, 29.11s/it]
 28%|██▊       | 118/420 [57:32<2:27:00, 29.21s/it]
 28%|██▊       | 119/420 [58:01<2:26:05, 29.12s/it]
 29%|██▊       | 120/420 [58:30<2:25:47, 29.16s/it]
 29%|██▉       | 121/420 [58:59<2:25:27, 29.19s/it]
 29%|██▉       | 122/420 [59:29<2:25:30, 29.30s/it]
 29%|██▉       | 123/420 [59:58<2:24:40, 29.23s/it]
 30%|██▉       | 124/420 [1:00:27<2:23:59, 29.19s/it]
 30%|██▉       | 125/420 [1:00:56<2:23:00, 29.08s/it]
 30%|███       | 126/420 [1:01:25<2:23:08, 29.21s/it]
 30%|███       | 127/420 [1:01:55<2:23:07, 29.31s/it]
 30%|███       | 128/420 [1:02:24<2:22:01, 29.18s/it]
 31%|███       | 129/420 [1:02:53<2:21:03, 29.08s/it]
 31%|███       | 130/420 [1:03:22<2:20:58, 29.17s/it]
 31%|███       | 131/420 [1:03:51<2:20:34, 29.18s/it]
 31%|███▏      | 132/420 [1:04:20<2:19:27, 29.06s/it]
 32%|███▏      | 133/420 [1:04:49<2:19:11, 29.10s/it]
 32%|███▏      | 134/420 [1:05:19<2:19:18, 29.22s/it]
 32%|███▏      | 135/420 [1:05:48<2:18:18, 29.12s/it]
 32%|███▏      | 136/420 [1:06:16<2:17:30, 29.05s/it]
 33%|███▎      | 137/420 [1:06:46<2:17:17, 29.11s/it]
 33%|███▎      | 138/420 [1:07:15<2:16:56, 29.14s/it]
 33%|███▎      | 139/420 [1:07:44<2:16:26, 29.13s/it]
 33%|███▎      | 140/420 [1:08:13<2:15:52, 29.11s/it]
 34%|███▎      | 141/420 [1:08:43<2:15:57, 29.24s/it]
 34%|███▍      | 142/420 [1:09:12<2:15:14, 29.19s/it]
 34%|███▍      | 143/420 [1:09:41<2:15:14, 29.29s/it]
 34%|███▍      | 144/420 [1:10:10<2:14:40, 29.28s/it]
 35%|███▍      | 145/420 [1:10:40<2:14:04, 29.25s/it]
 35%|███▍      | 146/420 [1:11:09<2:13:31, 29.24s/it]
 35%|███▌      | 147/420 [1:11:38<2:13:02, 29.24s/it]
 35%|███▌      | 148/420 [1:12:07<2:12:05, 29.14s/it]
 35%|███▌      | 149/420 [1:12:36<2:11:42, 29.16s/it]
 36%|███▌      | 150/420 [1:13:05<2:10:52, 29.08s/it]
                                                     
+
 36%|███▌      | 150/420 [1:13:05<2:10:52, 29.08s/it]
 36%|███▌      | 151/420 [1:13:35<2:10:59, 29.22s/it]
 36%|███▌      | 152/420 [1:14:04<2:10:30, 29.22s/it]
 36%|███▋      | 153/420 [1:14:33<2:10:28, 29.32s/it]
 37%|███▋      | 154/420 [1:15:03<2:09:49, 29.28s/it]
 37%|███▋      | 155/420 [1:15:32<2:09:14, 29.26s/it]
 37%|███▋      | 156/420 [1:16:01<2:08:15, 29.15s/it]
 37%|███▋      | 157/420 [1:16:30<2:07:44, 29.14s/it]
 38%|███▊      | 158/420 [1:16:59<2:07:20, 29.16s/it]
 38%|███▊      | 159/420 [1:17:29<2:07:20, 29.27s/it]
 38%|███▊      | 160/420 [1:17:58<2:06:20, 29.16s/it]
 38%|███▊      | 161/420 [1:18:27<2:05:54, 29.17s/it]
 39%|███▊      | 162/420 [1:18:56<2:05:17, 29.14s/it]
 39%|███▉      | 163/420 [1:19:25<2:04:54, 29.16s/it]
 39%|███▉      | 164/420 [1:19:55<2:04:54, 29.27s/it]
 39%|███▉      | 165/420 [1:20:23<2:03:59, 29.17s/it]
 40%|███▉      | 166/420 [1:20:52<2:03:10, 29.10s/it]
 40%|███▉      | 167/420 [1:21:21<2:02:32, 29.06s/it]
 40%|████      | 168/420 [1:21:50<2:01:55, 29.03s/it]
 40%|████      | 169/420 [1:22:20<2:02:04, 29.18s/it]
 40%|████      | 170/420 [1:22:49<2:01:39, 29.20s/it]
 41%|████      | 171/420 [1:23:18<2:01:11, 29.20s/it]
 41%|████      | 172/420 [1:23:47<2:00:21, 29.12s/it]
 41%|████      | 173/420 [1:24:16<1:59:59, 29.15s/it]
 41%|████▏     | 174/420 [1:24:46<1:59:58, 29.26s/it]
 42%|████▏     | 175/420 [1:25:15<1:59:28, 29.26s/it]
 42%|████▏     | 176/420 [1:25:45<1:59:07, 29.29s/it]
 42%|████▏     | 177/420 [1:26:14<1:58:58, 29.38s/it]
 42%|████▏     | 178/420 [1:26:44<1:58:42, 29.43s/it]
 43%|████▎     | 179/420 [1:27:13<1:57:56, 29.36s/it]
 43%|████▎     | 180/420 [1:27:42<1:57:18, 29.33s/it]
 43%|████▎     | 181/420 [1:28:11<1:56:42, 29.30s/it]
 43%|████▎     | 182/420 [1:28:41<1:56:07, 29.27s/it]
 44%|████▎     | 183/420 [1:29:10<1:55:57, 29.36s/it]
 44%|████▍     | 184/420 [1:29:40<1:55:39, 29.41s/it]
 44%|████▍     | 185/420 [1:30:09<1:54:35, 29.26s/it]
 44%|████▍     | 186/420 [1:30:38<1:54:03, 29.24s/it]
 45%|████▍     | 187/420 [1:31:07<1:53:33, 29.24s/it]
 45%|████▍     | 188/420 [1:31:36<1:53:05, 29.25s/it]
 45%|████▌     | 189/420 [1:32:06<1:52:59, 29.35s/it]
 45%|████▌     | 190/420 [1:32:35<1:52:43, 29.41s/it]
 45%|████▌     | 191/420 [1:33:04<1:51:31, 29.22s/it]
 46%|████▌     | 192/420 [1:33:33<1:51:00, 29.21s/it]
 46%|████▌     | 193/420 [1:34:03<1:50:31, 29.22s/it]
 46%|████▌     | 194/420 [1:34:32<1:50:01, 29.21s/it]
 46%|████▋     | 195/420 [1:35:01<1:49:55, 29.31s/it]
 47%|████▋     | 196/420 [1:35:31<1:49:22, 29.30s/it]
 47%|████▋     | 197/420 [1:36:00<1:48:27, 29.18s/it]
 47%|████▋     | 198/420 [1:36:29<1:48:23, 29.30s/it]
 47%|████▋     | 199/420 [1:36:59<1:48:12, 29.38s/it]
 48%|████▊     | 200/420 [1:37:27<1:47:04, 29.20s/it]
                                                     
+
 48%|████▊     | 200/420 [1:37:27<1:47:04, 29.20s/it]
 48%|████▊     | 201/420 [1:38:00<1:50:22, 30.24s/it]
 48%|████▊     | 202/420 [1:38:29<1:48:47, 29.94s/it]
 48%|████▊     | 203/420 [1:38:59<1:47:53, 29.83s/it]
 49%|████▊     | 204/420 [1:39:29<1:47:07, 29.76s/it]
 49%|████▉     | 205/420 [1:39:58<1:46:26, 29.70s/it]
 49%|████▉     | 206/420 [1:40:27<1:45:05, 29.46s/it]
 49%|████▉     | 207/420 [1:40:56<1:44:21, 29.40s/it]
 50%|████▉     | 208/420 [1:41:25<1:43:41, 29.34s/it]
 50%|████▉     | 209/420 [1:41:55<1:43:02, 29.30s/it]
 50%|█████     | 210/420 [1:42:24<1:42:48, 29.38s/it]
 50%|█████     | 211/420 [1:42:48<1:36:22, 27.67s/it]
 50%|█████     | 212/420 [1:43:17<1:37:32, 28.13s/it]
 51%|█████     | 213/420 [1:43:47<1:38:31, 28.56s/it]
 51%|█████     | 214/420 [1:44:16<1:38:46, 28.77s/it]
 51%|█████     | 215/420 [1:44:45<1:38:26, 28.81s/it]
 51%|█████▏    | 216/420 [1:45:14<1:38:04, 28.84s/it]
 52%|█████▏    | 217/420 [1:45:43<1:38:19, 29.06s/it]
 52%|█████▏    | 218/420 [1:46:13<1:38:19, 29.21s/it]
 52%|█████▏    | 219/420 [1:46:42<1:37:53, 29.22s/it]
 52%|█████▏    | 220/420 [1:47:11<1:37:05, 29.13s/it]
 53%|█████▎    | 221/420 [1:47:40<1:36:24, 29.07s/it]
 53%|█████▎    | 222/420 [1:48:10<1:36:25, 29.22s/it]
 53%|█████▎    | 223/420 [1:48:39<1:36:14, 29.31s/it]
 53%|█████▎    | 224/420 [1:49:08<1:35:42, 29.30s/it]
 54%|█████▎    | 225/420 [1:49:38<1:35:28, 29.37s/it]
 54%|█████▍    | 226/420 [1:50:07<1:34:32, 29.24s/it]
 54%|█████▍    | 227/420 [1:50:36<1:34:04, 29.24s/it]
 54%|█████▍    | 228/420 [1:51:05<1:33:34, 29.24s/it]
 55%|█████▍    | 229/420 [1:51:34<1:32:45, 29.14s/it]
 55%|█████▍    | 230/420 [1:52:03<1:32:03, 29.07s/it]
 55%|█████▌    | 231/420 [1:52:33<1:32:01, 29.21s/it]
 55%|█████▌    | 232/420 [1:53:02<1:31:52, 29.32s/it]
 55%|█████▌    | 233/420 [1:53:31<1:30:51, 29.15s/it]
 56%|█████▌    | 234/420 [1:54:00<1:30:27, 29.18s/it]
 56%|█████▌    | 235/420 [1:54:29<1:30:00, 29.19s/it]
 56%|█████▌    | 236/420 [1:54:58<1:29:14, 29.10s/it]
 56%|█████▋    | 237/420 [1:55:28<1:29:10, 29.24s/it]
 57%|█████▋    | 238/420 [1:55:57<1:28:41, 29.24s/it]
 57%|█████▋    | 239/420 [1:56:26<1:28:11, 29.24s/it]
 57%|█████▋    | 240/420 [1:56:56<1:27:58, 29.33s/it]
 57%|█████▋    | 241/420 [1:57:26<1:27:43, 29.40s/it]
 58%|█████▊    | 242/420 [1:57:55<1:27:05, 29.36s/it]
 58%|█████▊    | 243/420 [1:58:24<1:26:31, 29.33s/it]
 58%|█████▊    | 244/420 [1:58:53<1:25:39, 29.20s/it]
 58%|█████▊    | 245/420 [1:59:22<1:25:12, 29.21s/it]
 59%|█████▊    | 246/420 [1:59:51<1:24:25, 29.11s/it]
 59%|█████▉    | 247/420 [2:00:21<1:24:18, 29.24s/it]
 59%|█████▉    | 248/420 [2:00:50<1:23:48, 29.23s/it]
 59%|█████▉    | 249/420 [2:01:19<1:23:34, 29.32s/it]
 60%|█████▉    | 250/420 [2:01:49<1:23:01, 29.30s/it]
                                                     
+
 60%|█████▉    | 250/420 [2:01:49<1:23:01, 29.30s/it]
 60%|█████▉    | 251/420 [2:02:18<1:22:45, 29.38s/it]
 60%|██████    | 252/420 [2:02:47<1:22:09, 29.34s/it]
 60%|██████    | 253/420 [2:03:17<1:21:35, 29.31s/it]
 60%|██████    | 254/420 [2:03:46<1:21:02, 29.29s/it]
 61%|██████    | 255/420 [2:04:15<1:20:12, 29.17s/it]
 61%|██████    | 256/420 [2:04:44<1:19:46, 29.19s/it]
 61%|██████    | 257/420 [2:05:13<1:19:19, 29.20s/it]
 61%|██████▏   | 258/420 [2:05:43<1:19:06, 29.30s/it]
 62%|██████▏   | 259/420 [2:06:12<1:18:19, 29.19s/it]
 62%|██████▏   | 260/420 [2:06:41<1:17:37, 29.11s/it]
 62%|██████▏   | 261/420 [2:07:10<1:17:21, 29.19s/it]
 62%|██████▏   | 262/420 [2:07:39<1:16:39, 29.11s/it]
 63%|██████▎   | 263/420 [2:08:08<1:16:31, 29.24s/it]
 63%|██████▎   | 264/420 [2:08:38<1:15:54, 29.20s/it]
 63%|██████▎   | 265/420 [2:09:06<1:15:11, 29.10s/it]
 63%|██████▎   | 266/420 [2:09:36<1:14:47, 29.14s/it]
 64%|██████▎   | 267/420 [2:10:05<1:14:21, 29.16s/it]
 64%|██████▍   | 268/420 [2:10:34<1:13:57, 29.19s/it]
 64%|██████▍   | 269/420 [2:11:03<1:13:14, 29.10s/it]
 64%|██████▍   | 270/420 [2:11:33<1:13:05, 29.24s/it]
 65%|██████▍   | 271/420 [2:12:02<1:12:34, 29.23s/it]
 65%|██████▍   | 272/420 [2:12:31<1:12:19, 29.32s/it]
 65%|██████▌   | 273/420 [2:13:01<1:11:46, 29.29s/it]
 65%|██████▌   | 274/420 [2:13:30<1:11:15, 29.28s/it]
 65%|██████▌   | 275/420 [2:13:59<1:10:28, 29.17s/it]
 66%|██████▌   | 276/420 [2:14:28<1:10:03, 29.19s/it]
 66%|██████▌   | 277/420 [2:14:57<1:09:36, 29.21s/it]
 66%|██████▌   | 278/420 [2:15:27<1:09:21, 29.31s/it]
 66%|██████▋   | 279/420 [2:15:56<1:08:48, 29.28s/it]
 67%|██████▋   | 280/420 [2:16:25<1:08:23, 29.31s/it]
 67%|██████▋   | 281/420 [2:16:54<1:07:35, 29.17s/it]
 67%|██████▋   | 282/420 [2:17:23<1:07:07, 29.19s/it]
 67%|██████▋   | 283/420 [2:17:53<1:06:54, 29.30s/it]
 68%|██████▊   | 284/420 [2:18:22<1:06:08, 29.18s/it]
 68%|██████▊   | 285/420 [2:18:51<1:05:55, 29.30s/it]
 68%|██████▊   | 286/420 [2:19:21<1:05:19, 29.25s/it]
 68%|██████▊   | 287/420 [2:19:49<1:04:31, 29.11s/it]
 69%|██████▊   | 288/420 [2:20:18<1:03:48, 29.01s/it]
 69%|██████▉   | 289/420 [2:20:47<1:03:27, 29.06s/it]
 69%|██████▉   | 290/420 [2:21:16<1:02:59, 29.07s/it]
 69%|██████▉   | 291/420 [2:21:46<1:02:37, 29.13s/it]
 70%|██████▉   | 292/420 [2:22:15<1:02:26, 29.27s/it]
 70%|██████▉   | 293/420 [2:22:44<1:01:53, 29.24s/it]
 70%|███████   | 294/420 [2:23:14<1:01:25, 29.25s/it]
 70%|███████   | 295/420 [2:23:43<1:00:51, 29.21s/it]
 70%|███████   | 296/420 [2:24:12<1:00:15, 29.16s/it]
 71%|███████   | 297/420 [2:24:41<1:00:00, 29.28s/it]
 71%|███████   | 298/420 [2:25:11<59:41, 29.35s/it]  
 71%|███████   | 299/420 [2:25:41<59:18, 29.41s/it]
 71%|███████▏  | 300/420 [2:26:09<58:30, 29.26s/it]
                                                   
+
 71%|███████▏  | 300/420 [2:26:09<58:30, 29.26s/it]
 72%|███████▏  | 301/420 [2:26:39<58:12, 29.35s/it]
 72%|███████▏  | 302/420 [2:27:08<57:35, 29.28s/it]
 72%|███████▏  | 303/420 [2:27:37<57:03, 29.26s/it]
 72%|███████▏  | 304/420 [2:28:06<56:22, 29.16s/it]
 73%|███████▎  | 305/420 [2:28:35<55:55, 29.18s/it]
 73%|███████▎  | 306/420 [2:29:04<55:17, 29.10s/it]
 73%|███████▎  | 307/420 [2:29:34<55:02, 29.22s/it]
 73%|███████▎  | 308/420 [2:30:03<54:44, 29.33s/it]
 74%|███████▎  | 309/420 [2:30:33<54:23, 29.40s/it]
 74%|███████▍  | 310/420 [2:31:02<53:47, 29.34s/it]
 74%|███████▍  | 311/420 [2:31:31<53:03, 29.21s/it]
 74%|███████▍  | 312/420 [2:32:00<52:36, 29.22s/it]
 75%|███████▍  | 313/420 [2:32:30<52:18, 29.33s/it]
 75%|███████▍  | 314/420 [2:32:59<51:45, 29.30s/it]
 75%|███████▌  | 315/420 [2:33:28<51:03, 29.18s/it]
 75%|███████▌  | 316/420 [2:33:57<50:36, 29.19s/it]
 75%|███████▌  | 317/420 [2:34:27<50:08, 29.21s/it]
 76%|███████▌  | 318/420 [2:34:56<49:40, 29.22s/it]
 76%|███████▌  | 319/420 [2:35:25<49:21, 29.32s/it]
 76%|███████▌  | 320/420 [2:35:54<48:44, 29.24s/it]
 76%|███████▋  | 321/420 [2:36:23<47:58, 29.07s/it]
 77%|███████▋  | 322/420 [2:36:52<47:34, 29.12s/it]
 77%|███████▋  | 323/420 [2:37:21<46:59, 29.06s/it]
 77%|███████▋  | 324/420 [2:37:51<46:34, 29.11s/it]
 77%|███████▋  | 325/420 [2:38:20<46:18, 29.24s/it]
 78%|███████▊  | 326/420 [2:38:50<45:57, 29.34s/it]
 78%|███████▊  | 327/420 [2:39:19<45:26, 29.32s/it]
 78%|███████▊  | 328/420 [2:39:48<44:45, 29.19s/it]
 78%|███████▊  | 329/420 [2:40:17<44:17, 29.20s/it]
 79%|███████▊  | 330/420 [2:40:46<43:39, 29.11s/it]
 79%|███████▉  | 331/420 [2:41:15<43:01, 29.00s/it]
 79%|███████▉  | 332/420 [2:41:44<42:46, 29.17s/it]
 79%|███████▉  | 333/420 [2:42:14<42:24, 29.24s/it]
 80%|███████▉  | 334/420 [2:42:43<42:03, 29.34s/it]
 80%|███████▉  | 335/420 [2:43:12<41:31, 29.31s/it]
 80%|████████  | 336/420 [2:43:42<40:59, 29.28s/it]
 80%|████████  | 337/420 [2:44:11<40:28, 29.26s/it]
 80%|████████  | 338/420 [2:44:40<39:57, 29.24s/it]
 81%|████████  | 339/420 [2:45:09<39:19, 29.12s/it]
 81%|████████  | 340/420 [2:45:38<38:41, 29.02s/it]
 81%|████████  | 341/420 [2:46:07<38:17, 29.08s/it]
 81%|████████▏ | 342/420 [2:46:36<37:51, 29.12s/it]
 82%|████████▏ | 343/420 [2:47:05<37:17, 29.06s/it]
 82%|████████▏ | 344/420 [2:47:34<36:52, 29.12s/it]
 82%|████████▏ | 345/420 [2:48:04<36:26, 29.15s/it]
 82%|████████▏ | 346/420 [2:48:32<35:51, 29.07s/it]
 83%|████████▎ | 347/420 [2:49:02<35:25, 29.11s/it]
 83%|████████▎ | 348/420 [2:49:31<35:06, 29.26s/it]
 83%|████████▎ | 349/420 [2:50:00<34:37, 29.26s/it]
 83%|████████▎ | 350/420 [2:50:30<34:07, 29.25s/it]
                                                   
+
 83%|████████▎ | 350/420 [2:50:30<34:07, 29.25s/it]
 84%|████████▎ | 351/420 [2:50:59<33:37, 29.24s/it]
 84%|████████▍ | 352/420 [2:51:28<33:00, 29.13s/it]
 84%|████████▍ | 353/420 [2:51:57<32:40, 29.26s/it]
 84%|████████▍ | 354/420 [2:52:26<32:07, 29.20s/it]
 85%|████████▍ | 355/420 [2:52:56<31:38, 29.21s/it]
 85%|████████▍ | 356/420 [2:53:25<31:15, 29.30s/it]
 85%|████████▌ | 357/420 [2:53:55<30:50, 29.37s/it]
 85%|████████▌ | 358/420 [2:54:24<30:18, 29.34s/it]
 85%|████████▌ | 359/420 [2:54:54<29:53, 29.40s/it]
 86%|████████▌ | 360/420 [2:55:23<29:26, 29.45s/it]
 86%|████████▌ | 361/420 [2:55:52<28:48, 29.29s/it]
 86%|████████▌ | 362/420 [2:56:21<28:17, 29.27s/it]
 86%|████████▋ | 363/420 [2:56:51<27:53, 29.35s/it]
 87%|████████▋ | 364/420 [2:57:20<27:27, 29.42s/it]
 87%|████████▋ | 365/420 [2:57:49<26:49, 29.26s/it]
 87%|████████▋ | 366/420 [2:58:18<26:14, 29.16s/it]
 87%|████████▋ | 367/420 [2:58:47<25:38, 29.03s/it]
 88%|████████▊ | 368/420 [2:59:16<25:08, 29.01s/it]
 88%|████████▊ | 369/420 [2:59:45<24:47, 29.17s/it]
 88%|████████▊ | 370/420 [3:00:15<24:19, 29.18s/it]
 88%|████████▊ | 371/420 [3:00:44<23:55, 29.29s/it]
 89%|████████▊ | 372/420 [3:01:13<23:18, 29.13s/it]
 89%|████████▉ | 373/420 [3:01:42<22:50, 29.16s/it]
 89%|████████▉ | 374/420 [3:02:11<22:22, 29.18s/it]
 89%|████████▉ | 375/420 [3:02:41<21:53, 29.18s/it]
 90%|████████▉ | 376/420 [3:03:10<21:24, 29.19s/it]
 90%|████████▉ | 377/420 [3:03:39<20:54, 29.17s/it]
 90%|█████████ | 378/420 [3:04:08<20:29, 29.28s/it]
 90%|█████████ | 379/420 [3:04:37<19:54, 29.13s/it]
 90%|█████████ | 380/420 [3:05:06<19:26, 29.17s/it]
 91%|█████████ | 381/420 [3:05:36<18:58, 29.19s/it]
 91%|█████████ | 382/420 [3:06:05<18:27, 29.15s/it]
 91%|█████████ | 383/420 [3:06:34<17:59, 29.18s/it]
 91%|█████████▏| 384/420 [3:07:03<17:30, 29.19s/it]
 92%|█████████▏| 385/420 [3:07:32<16:59, 29.12s/it]
 92%|█████████▏| 386/420 [3:08:02<16:34, 29.25s/it]
 92%|█████████▏| 387/420 [3:08:31<16:05, 29.24s/it]
 92%|█████████▏| 388/420 [3:09:00<15:35, 29.24s/it]
 93%|█████████▎| 389/420 [3:09:29<15:06, 29.24s/it]
 93%|█████████▎| 390/420 [3:09:58<14:33, 29.13s/it]
 93%|█████████▎| 391/420 [3:10:28<14:05, 29.16s/it]
 93%|█████████▎| 392/420 [3:10:57<13:37, 29.18s/it]
 94%|█████████▎| 393/420 [3:11:26<13:08, 29.20s/it]
 94%|█████████▍| 394/420 [3:11:55<12:39, 29.21s/it]
 94%|█████████▍| 395/420 [3:12:24<12:10, 29.21s/it]
 94%|█████████▍| 396/420 [3:12:54<11:40, 29.18s/it]
 95%|█████████▍| 397/420 [3:13:22<11:09, 29.10s/it]
 95%|█████████▍| 398/420 [3:13:52<10:43, 29.23s/it]
 95%|█████████▌| 399/420 [3:14:21<10:14, 29.24s/it]
 95%|█████████▌| 400/420 [3:14:51<09:46, 29.34s/it]
                                                   
+
 95%|█████████▌| 400/420 [3:14:51<09:46, 29.34s/it]
 95%|█████████▌| 401/420 [3:15:24<09:36, 30.35s/it]
 96%|█████████▌| 402/420 [3:15:53<09:01, 30.10s/it]
 96%|█████████▌| 403/420 [3:16:23<08:28, 29.93s/it]
 96%|█████████▌| 404/420 [3:16:51<07:54, 29.63s/it]
 96%|█████████▋| 405/420 [3:17:21<07:24, 29.60s/it]
 97%|█████████▋| 406/420 [3:17:50<06:50, 29.35s/it]
 97%|█████████▋| 407/420 [3:18:19<06:19, 29.18s/it]
 97%|█████████▋| 408/420 [3:18:48<05:50, 29.20s/it]
 97%|█████████▋| 409/420 [3:19:17<05:21, 29.21s/it]
 98%|█████████▊| 410/420 [3:19:46<04:52, 29.21s/it]
 98%|█████████▊| 411/420 [3:20:16<04:23, 29.30s/it]
 98%|█████████▊| 412/420 [3:20:45<03:53, 29.17s/it]
 98%|█████████▊| 413/420 [3:21:14<03:25, 29.29s/it]
 99%|█████████▊| 414/420 [3:21:43<02:55, 29.18s/it]
 99%|█████████▉| 415/420 [3:22:12<02:25, 29.10s/it]
 99%|█████████▉| 416/420 [3:22:42<01:56, 29.23s/it]
 99%|█████████▉| 417/420 [3:23:11<01:27, 29.14s/it]
                                                   
+
+***** train metrics *****
+  epoch                    =        2.0
+  train_loss               =      1.281
+  train_runtime            = 3:24:38.74
+  train_samples_per_second =      0.411
+  train_steps_per_second   =      0.034

without_race_ft/train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.0,
+    "train_loss": 1.2810182730356852,
+    "train_runtime": 12278.7402,
+    "train_samples_per_second": 0.411,
+    "train_steps_per_second": 0.034
+}

without_race_ft/trainer_state.json

@@ -0,0 +1,84 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.995249406175772,
+  "eval_steps": 500,
+  "global_step": 420,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.24,
+      "global_step": 50,
+      "learning_rate": 0.0001,
+      "loss": 7.3699,
+      "step": 50
+    },
+    {
+      "epoch": 0.48,
+      "global_step": 100,
+      "learning_rate": 0.0001,
+      "loss": 0.6661,
+      "step": 100
+    },
+    {
+      "epoch": 0.71,
+      "global_step": 150,
+      "learning_rate": 0.0001,
+      "loss": 0.5816,
+      "step": 150
+    },
+    {
+      "epoch": 0.95,
+      "global_step": 200,
+      "learning_rate": 0.0001,
+      "loss": 0.5427,
+      "step": 200
+    },
+    {
+      "epoch": 1.19,
+      "global_step": 250,
+      "learning_rate": 0.0001,
+      "loss": 0.4052,
+      "step": 250
+    },
+    {
+      "epoch": 1.43,
+      "global_step": 300,
+      "learning_rate": 0.0001,
+      "loss": 0.3372,
+      "step": 300
+    },
+    {
+      "epoch": 1.66,
+      "global_step": 350,
+      "learning_rate": 0.0001,
+      "loss": 0.3266,
+      "step": 350
+    },
+    {
+      "epoch": 1.9,
+      "global_step": 400,
+      "learning_rate": 0.0001,
+      "loss": 0.3891,
+      "step": 400
+    },
+    {
+      "epoch": 2.0,
+      "step": 420,
+      "total_flos": 2.242379833713623e+17,
+      "train_loss": 1.2810182730356852,
+      "train_runtime": 12278.7402,
+      "train_samples_per_second": 0.411,
+      "train_steps_per_second": 0.034
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 420,
+  "num_train_epochs": 2,
+  "save_steps": 200,
+  "total_flos": 2.242379833713623e+17,
+  "trial_name": null,
+  "trial_params": null
+}

without_race_ft/without_race_ft.json

@@ -0,0 +1,31 @@
+{
+    "desc": "不在race数据集上预训练,仅使用quality数据集训练",
+    "output_dir": "output/ablation_study/QuALITY/without_race_ft",
+    "model_name_or_path": "/data0/maqi/huggingface_models/llama-2-7b",
+    "train_file": "/data0/maqi/KGLQA-data/datasets/QuALITY/quality_rocketqa_2048_instruct/train.jsonl",
+    "num_train_epochs": 2,
+    "per_device_train_batch_size": 6,
+    "gradient_accumulation_steps": 2,
+    "learning_rate": 1e-4,
+    "max_seq_length": 2048,
+    "logging_steps": 50,
+    "save_steps": 200,
+    "save_total_limit": 1,
+    "lr_scheduler_type": "constant_with_warmup",
+    "warmup_ratio": 0.1,
+    "lora_rank": 128,
+    "lora_alpha": 64,
+    "lora_dropout": 0.05,
+
+    "gradient_checkpointing": true,
+    "disable_tqdm": false,
+    "optim": "paged_adamw_32bit",
+    "seed": 318,
+    "fp16": true,
+    "report_to": "tensorboard",
+    "dataloader_num_workers": 10,
+    "save_strategy": "steps",
+    "weight_decay": 0,
+    "max_grad_norm": 0.3,
+    "remove_unused_columns": false
+}