Training in progress, step 7000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13e8bfb6cc0a5778a11cf6475ff56b532497650ddb8b982d4a541b76fea9f445
+oid sha256:2833606f70f74be87bdb4bbe4cf8557b7e259740d4b73b3931cfcfa776d5dc23
 size 3219908024

wandb/run-20231118_132020-qg88991p/files/output.log

@@ -6505,3 +6505,1068 @@ Reading metadata...: 1it [00:00,  6.69it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▉                                                                                                                                                                                                                    | 6080/60000 [18:03:49<136:45:44,  9.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▉                                                                                                                                                                                                                    | 6099/60000 [18:06:46<138:57:15,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████                                                                                                                                                                                                                    | 6120/60000 [18:10:02<137:38:52,  9.20s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▏                                                                                                                                                                                                                   | 6140/60000 [18:13:08<141:54:36,  9.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▏                                                                                                                                                                                                                   | 6159/60000 [18:16:10<139:33:45,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▎                                                                                                                                                                                                                   | 6179/60000 [18:19:14<140:41:45,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▍                                                                                                                                                                                                                   | 6199/60000 [18:22:21<142:31:31,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▍                                                                                                                                                                                                                   | 6220/60000 [18:25:33<136:35:07,  9.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▌                                                                                                                                                                                                                   | 6239/60000 [18:28:35<146:24:14,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▌                                                                                                                                                                                                                   | 6260/60000 [18:31:48<135:03:47,  9.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▋                                                                                                                                                                                                                   | 6280/60000 [18:34:52<134:42:02,  9.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|████████████████████████▊                                                                                                                                                                                                                   | 6300/60000 [18:38:14<178:07:41, 11.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|████████████████████████▊                                                                                                                                                                                                                   | 6319/60000 [18:41:12<136:30:04,  9.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|████████████████████████▉                                                                                                                                                                                                                   | 6339/60000 [18:44:56<137:07:25,  9.20s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████                                                                                                                                                                                                                   | 6360/60000 [18:48:09<137:16:45,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████                                                                                                                                                                                                                   | 6380/60000 [18:51:13<136:51:39,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▏                                                                                                                                                                                                                  | 6400/60000 [18:54:18<137:26:33,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▎                                                                                                                                                                                                                  | 6420/60000 [18:57:26<137:32:23,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▎                                                                                                                                                                                                                  | 6440/60000 [19:00:28<135:19:39,  9.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▍                                                                                                                                                                                                                  | 6460/60000 [19:03:35<153:07:22, 10.30s/it]
+
+
+
+
+Reading metadata...: 2165it [00:00, 12982.32it/s]                                                                                                                                                                                                | 6464/60000 [19:04:18<155:13:19, 10.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▍                                                                                                                                                                                                                  | 6480/60000 [19:06:54<143:38:53,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▌                                                                                                                                                                                                                  | 6500/60000 [19:10:07<148:01:49,  9.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▋                                                                                                                                                                                                                  | 6520/60000 [19:13:16<142:11:29,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▋                                                                                                                                                                                                                  | 6540/60000 [19:16:23<143:35:52,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▊                                                                                                                                                                                                                  | 6560/60000 [19:19:28<136:50:54,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|█████████████████████████▉                                                                                                                                                                                                                  | 6580/60000 [19:22:33<138:23:59,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|████████████��████████████▉                                                                                                                                                                                                                  | 6600/60000 [19:26:09<137:35:52,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████                                                                                                                                                                                                                  | 6620/60000 [19:29:13<135:27:06,  9.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████                                                                                                                                                                                                                  | 6640/60000 [19:33:06<143:51:38,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▏                                                                                                                                                                                                                 | 6659/60000 [19:36:01<136:15:03,  9.20s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▎                                                                                                                                                                                                                 | 6679/60000 [19:39:12<144:17:09,  9.74s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▎                                                                                                                                                                                                                 | 6699/60000 [19:42:18<133:33:33,  9.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▍                                                                                                                                                                                                                 | 6719/60000 [19:45:22<137:27:06,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▌                                                                                                                                                                                                                 | 6740/60000 [19:48:49<143:07:45,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▌                                                                                                                                                                                                                 | 6760/60000 [19:51:53<137:12:21,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▋                                                                                                                                                                                                                 | 6779/60000 [19:54:52<135:29:24,  9.16s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▋                                                                                                                                                                                                                 | 6799/60000 [19:57:56<135:05:53,  9.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▊                                                                                                                                                                                                                 | 6819/60000 [20:00:59<137:10:24,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▉                                                                                                                                                                                                                 | 6839/60000 [20:04:03<133:52:03,  9.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|██████████████████████████▉                                                                                                                                                                                                                 | 6860/60000 [20:07:22<134:15:04,  9.09s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 11%|███████████████████████████                                                                                                                                                                                                                 | 6880/60000 [20:10:28<141:11:17,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 4019.95it/s]                                                                                                                                                                                                 | 6899/60000 [20:13:26<135:28:47,  9.18s/it]
+
+ 12%|███████████████████████████▏                                                                                                                                                                                                                | 6900/60000 [20:13:37<143:28:17,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▏                                                                                                                                                                                                                | 6919/60000 [20:16:36<138:08:56,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▎                                                                                                                                                                                                                | 6940/60000 [20:19:57<155:15:25, 10.53s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▎                                                                                                                                                                                                                | 6959/60000 [20:22:53<136:55:04,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▍                                                                                                                                                                                                                | 6979/60000 [20:26:13<139:06:06,  9.44s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▌                                                                                                                                                                                                                | 7000/60000 [20:29:27<135:59:18,  9.24s/it][INFO|trainer.py:3173] 2023-11-19 09:49:51,427 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-19 09:49:51,427 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-19 09:49:51,428 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 10117.74it/s]
+Reading metadata...: 1it [00:00,  6.90it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-19 09:49:52,356 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+ 12%|███████████████████████████▌                                                                                                                                                                                                                | 7000/60000 [20:40:36<135:59:18,  9.24s/it]
+ 12%|███████████████████████████▌                                                                                                                                                                                                                | 7000/60000 [20:40:36<135:59:18,  9.24s/it][INFO|trainer.py:2896] 2023-11-19 10:01:42,953 >> Saving model checkpoint to ./checkpoint-7000
+[INFO|configuration_utils.py:462] 2023-11-19 10:01:42,964 >> Configuration saved in ./checkpoint-7000/config.json
+[INFO|configuration_utils.py:568] 2023-11-19 10:01:42,971 >> Configuration saved in ./checkpoint-7000/generation_config.json
+[2023-11-19 10:02:35,205] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step7000 is about to be saved!
+[2023-11-19 10:02:35,238] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt
+[2023-11-19 10:02:35,238] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-19 10:02:35,175 >> Model weights saved in ./checkpoint-7000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-19 10:02:35,181 >> Feature extractor saved in ./checkpoint-7000/preprocessor_config.json
+[2023-11-19 10:02:56,297] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-7000/global_step7000/mp_rank_00_model_states.pt.
+[2023-11-19 10:02:56,319] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-19 10:04:10,359] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-19 10:04:10,369] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-7000/global_step7000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-19 10:04:10,370] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step7000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-19 10:05:15,419 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▌                                                                                                                                                                                                                | 7020/60000 [20:48:31<159:37:27, 10.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▋                                                                                                                                                                                                                | 7040/60000 [20:52:05<149:28:41, 10.16s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 12%|███████████████████████████▊                                                                                                                                                                                                                | 7059/60000 [20:55:20<151:14:07, 10.28s/it]
+
+
+
+
+
+

wandb/run-20231118_132020-qg88991p/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0959, "train/learning_rate": 2.7432711864406777e-06, "train/epoch": 0.1, "train/global_step": 6060, "_timestamp": 1700371265.4985242, "_runtime": 64844.58343410492, "_step": 308, "eval/loss": 0.1368408203125, "eval/wer": 8.876743309460988, "eval/runtime": 591.8673, "eval/samples_per_second": 2.879, "eval/steps_per_second": 0.72}
+{"train/loss": 0.1191, "train/learning_rate": 2.6924237288135594e-06, "train/epoch": 0.12, "train/global_step": 7060, "_timestamp": 1700381754.2567582, "_runtime": 75333.34166812897, "_step": 359, "eval/loss": 0.1439208984375, "eval/wer": 8.660007538635508, "eval/runtime": 668.8365, "eval/samples_per_second": 2.548, "eval/steps_per_second": 0.637}

wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:45abb02b364f3abe2253240c15f9ce34b371ded1f82652adbb3b0ae2cfbbceb2
-size 5049554
+oid sha256:829e7e0b338083a1d991e646ecb5258a236916e87ee43a0b5f00c5dcaf0766fe
+size 5909427