Training in progress, epoch 3

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d931da261693711d55a7459f1e825fc4ae610361ff30ad7d572ceb3c42c68e4
+oid sha256:4e47634cebdf0b5c89da0ff9e64965772f4e66d2b70fe0118fe63c45d4c004cf
 size 94409168

debug.log

@@ -1075,3 +1075,428 @@ trainable params: 23,592,960 || all params: 797,623,040 || trainable%: 2.9579
 
 67% 366/549 [36:55<14:09,  4.64s/it]
 
 
                                 [2025-10-10 13:48:46,376] [INFO] [axolotl.core.trainers.base._save:671] [PID:24741] Saving model checkpoint to ./qlora-out/checkpoint-366
+
 67% 367/549 [37:06<31:54, 10.52s/it]
 67% 368/549 [37:10<25:42,  8.52s/it]
 67% 369/549 [37:13<20:50,  6.95s/it]
 67% 370/549 [37:17<17:59,  6.03s/it]
 68% 371/549 [37:21<16:00,  5.39s/it]
 68% 372/549 [37:25<14:37,  4.96s/it][2025-10-10 13:49:16,639] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:49:19,397] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2493188381195068
+[2025-10-10 13:49:20,670] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2731273174285889
+[2025-10-10 13:49:21,916] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2452585697174072
+[2025-10-10 13:49:23,190] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2739589214324951
+[2025-10-10 13:49:23,190] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.83it/s]
+
 75% 3/4 [00:02<00:00,  1.31it/s]
+
                                     
+
                                 
+
 68% 372/549 [37:37<14:37,  4.96s/it]
+
+
                                 
 68% 373/549 [37:41<24:29,  8.35s/it]
 68% 374/549 [37:45<20:27,  7.01s/it]
 68% 375/549 [37:49<17:38,  6.08s/it]
 68% 376/549 [37:53<15:38,  5.43s/it]
 69% 377/549 [37:57<14:13,  4.96s/it]
 69% 378/549 [38:01<13:13,  4.64s/it][2025-10-10 13:49:52,383] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:49:55,178] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2608587741851807
+[2025-10-10 13:49:56,450] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2715425491333008
+[2025-10-10 13:49:57,715] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2646293640136719
+[2025-10-10 13:49:59,010] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2951922416687012
+[2025-10-10 13:49:59,010] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 69% 378/549 [38:13<13:13,  4.64s/it]
+
+
                                 
 69% 379/549 [38:17<23:03,  8.14s/it]
 69% 380/549 [38:21<19:17,  6.85s/it]
 69% 381/549 [38:25<16:40,  5.95s/it]
 70% 382/549 [38:29<14:50,  5.33s/it]
 70% 383/549 [38:32<13:32,  4.90s/it]
 70% 384/549 [38:36<12:39,  4.60s/it][2025-10-10 13:50:28,063] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:50:31,217] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2841885089874268
+[2025-10-10 13:50:32,493] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2753474712371826
+[2025-10-10 13:50:33,766] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2733006477355957
+[2025-10-10 13:50:35,051] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2847979068756104
+[2025-10-10 13:50:35,051] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                     
+
                                 
+
 70% 384/549 [38:49<12:39,  4.60s/it]
+
+
                                 
 70% 385/549 [38:53<22:27,  8.22s/it]
 70% 386/549 [38:57<18:46,  6.91s/it]
 70% 387/549 [39:01<16:14,  6.01s/it]
 71% 388/549 [39:05<14:26,  5.38s/it]
 71% 389/549 [39:09<13:10,  4.94s/it]
 71% 390/549 [39:13<12:16,  4.63s/it][2025-10-10 13:51:04,231] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:51:07,499] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5531108379364014
+[2025-10-10 13:51:08,778] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2783293724060059
+[2025-10-10 13:51:10,059] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2812681198120117
+[2025-10-10 13:51:11,349] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.28898286819458
+[2025-10-10 13:51:11,349] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 71% 390/549 [39:25<12:16,  4.63s/it]
+
+
                                 
 71% 391/549 [39:29<21:45,  8.27s/it]
 71% 392/549 [39:33<18:09,  6.94s/it]
 72% 393/549 [39:37<15:38,  6.01s/it]
 72% 394/549 [39:41<13:52,  5.37s/it]
 72% 395/549 [39:45<12:37,  4.92s/it]
 72% 396/549 [39:49<11:45,  4.61s/it][2025-10-10 13:51:40,295] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:51:43,495] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8327646255493164
+[2025-10-10 13:51:44,795] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2993097305297852
+[2025-10-10 13:51:46,167] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.372373342514038
+[2025-10-10 13:51:47,442] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2747223377227783
+[2025-10-10 13:51:47,442] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 72% 396/549 [40:02<11:45,  4.61s/it]
+
+
                                 
 72% 397/549 [40:05<20:55,  8.26s/it]
 72% 398/549 [40:09<17:27,  6.94s/it]
 73% 399/549 [40:13<15:02,  6.02s/it]
 73% 400/549 [40:17<13:22,  5.38s/it]
                                     
+
 73% 400/549 [40:17<13:22,  5.38s/it]
 73% 401/549 [40:21<12:11,  4.94s/it]
 73% 402/549 [40:25<11:21,  4.63s/it][2025-10-10 13:52:16,526] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:52:19,410] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5968072414398193
+[2025-10-10 13:52:21,081] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.670398473739624
+[2025-10-10 13:52:22,353] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.272465705871582
+[2025-10-10 13:52:23,640] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.285794973373413
+[2025-10-10 13:52:23,640] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 73% 402/549 [40:38<11:21,  4.63s/it]
+
+
                                 
 73% 403/549 [40:42<20:07,  8.27s/it]
 74% 404/549 [40:45<16:47,  6.95s/it]
 74% 405/549 [40:49<14:28,  6.03s/it]
 74% 406/549 [40:53<12:52,  5.40s/it]
 74% 407/549 [40:57<11:43,  4.95s/it]
 74% 408/549 [41:01<10:53,  4.63s/it][2025-10-10 13:52:52,766] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:52:55,394] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.33892822265625
+[2025-10-10 13:52:57,235] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.840940237045288
+[2025-10-10 13:52:58,502] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2660002708435059
+[2025-10-10 13:52:59,779] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2767868041992188
+[2025-10-10 13:52:59,779] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 74% 408/549 [41:14<10:53,  4.63s/it]
+
+
                                 
 74% 409/549 [41:18<19:13,  8.24s/it]
 75% 410/549 [41:22<16:02,  6.93s/it]
 75% 411/549 [41:25<13:48,  6.01s/it]
 75% 412/549 [41:29<12:16,  5.38s/it]
 75% 413/549 [41:33<11:11,  4.94s/it]
 75% 414/549 [41:37<10:24,  4.63s/it][2025-10-10 13:53:28,861] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:53:31,446] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2704637050628662
+[2025-10-10 13:53:32,999] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.552428960800171
+[2025-10-10 13:53:34,672] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6721296310424805
+[2025-10-10 13:53:35,947] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.274794101715088
+[2025-10-10 13:53:35,947] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 75% 414/549 [41:50<10:24,  4.63s/it]
+
+
                                 
 76% 415/549 [41:54<18:28,  8.27s/it]
 76% 416/549 [41:58<15:24,  6.95s/it]
 76% 417/549 [42:02<13:15,  6.03s/it]
 76% 418/549 [42:06<11:46,  5.39s/it]
 76% 419/549 [42:10<10:43,  4.95s/it]
 77% 420/549 [42:13<09:58,  4.64s/it][2025-10-10 13:54:05,114] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:54:07,798] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3199918270111084
+[2025-10-10 13:54:09,209] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4115185737609863
+[2025-10-10 13:54:11,055] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8450191020965576
+[2025-10-10 13:54:12,332] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2769029140472412
+[2025-10-10 13:54:12,332] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 77% 420/549 [42:27<09:58,  4.64s/it]
+
+
                                 
 77% 421/549 [42:30<17:45,  8.32s/it]
 77% 422/549 [42:34<14:46,  6.98s/it]
 77% 423/549 [42:38<12:41,  6.04s/it]
 77% 424/549 [42:42<11:15,  5.40s/it]
 77% 425/549 [42:46<10:13,  4.95s/it]
 78% 426/549 [42:50<09:30,  4.63s/it][2025-10-10 13:54:41,441] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:54:44,036] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2831828594207764
+[2025-10-10 13:54:45,310] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2733635902404785
+[2025-10-10 13:54:47,008] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.697882890701294
+[2025-10-10 13:54:48,636] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6273066997528076
+[2025-10-10 13:54:48,636] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 78% 426/549 [43:03<09:30,  4.63s/it]
+
+
                                 
 78% 427/549 [43:07<16:53,  8.31s/it]
 78% 428/549 [43:11<14:04,  6.98s/it]
 78% 429/549 [43:14<12:05,  6.05s/it]
 78% 430/549 [43:18<10:43,  5.40s/it]
 79% 431/549 [43:22<09:44,  4.96s/it]
 79% 432/549 [43:26<09:02,  4.64s/it][2025-10-10 13:55:17,787] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:55:20,430] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3111767768859863
+[2025-10-10 13:55:21,756] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3263280391693115
+[2025-10-10 13:55:23,205] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4486384391784668
+[2025-10-10 13:55:25,025] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8193638324737549
+[2025-10-10 13:55:25,025] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 79% 432/549 [43:39<09:02,  4.64s/it]
+
+
                                 
 79% 433/549 [43:43<16:05,  8.32s/it]
 79% 434/549 [43:47<13:22,  6.98s/it]
 79% 435/549 [43:51<11:28,  6.04s/it]
 79% 436/549 [43:55<10:09,  5.39s/it]
 80% 437/549 [43:59<09:13,  4.94s/it]
 80% 438/549 [44:02<08:33,  4.63s/it][2025-10-10 13:55:54,066] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:55:56,645] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.282938003540039
+[2025-10-10 13:55:57,916] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2711057662963867
+[2025-10-10 13:55:59,205] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2884700298309326
+[2025-10-10 13:56:00,852] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6469426155090332
+[2025-10-10 13:56:00,852] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 80% 438/549 [44:15<08:33,  4.63s/it]
+
+
                                 
 80% 439/549 [44:19<15:09,  8.27s/it]
 80% 440/549 [44:23<12:36,  6.94s/it]
 80% 441/549 [44:27<10:50,  6.03s/it]
 81% 442/549 [44:31<09:36,  5.39s/it]
 81% 443/549 [44:35<08:44,  4.95s/it]
 81% 444/549 [44:39<08:06,  4.63s/it][2025-10-10 13:56:30,285] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:56:32,867] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.277099609375
+[2025-10-10 13:56:34,157] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2903683185577393
+[2025-10-10 13:56:35,441] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2831330299377441
+[2025-10-10 13:56:36,811] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3701646327972412
+[2025-10-10 13:56:36,811] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 81% 444/549 [44:51<08:06,  4.63s/it]
+
+
                                 
 81% 445/549 [44:55<14:18,  8.26s/it]
 81% 446/549 [44:59<11:55,  6.95s/it]
 81% 447/549 [45:03<10:14,  6.03s/it]
 82% 448/549 [45:07<09:03,  5.38s/it]
 82% 449/549 [45:11<08:14,  4.94s/it]
 82% 450/549 [45:15<07:38,  4.63s/it][2025-10-10 13:57:06,458] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:57:09,048] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2799561023712158
+[2025-10-10 13:57:10,351] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3030431270599365
+[2025-10-10 13:57:11,608] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2559781074523926
+[2025-10-10 13:57:12,875] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2669663429260254
+[2025-10-10 13:57:12,875] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 82% 450/549 [45:27<07:38,  4.63s/it]
+
+
                                 
 82% 451/549 [45:31<13:21,  8.18s/it]
 82% 452/549 [45:35<11:08,  6.89s/it]
 83% 453/549 [45:39<09:34,  5.98s/it]
 83% 454/549 [45:43<08:28,  5.35s/it]
 83% 455/549 [45:47<07:43,  4.93s/it]
 83% 456/549 [45:51<07:10,  4.63s/it][2025-10-10 13:57:42,418] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:57:45,040] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2982087135314941
+[2025-10-10 13:57:46,346] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3049485683441162
+[2025-10-10 13:57:47,647] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3008086681365967
+[2025-10-10 13:57:48,951] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3040728569030762
+[2025-10-10 13:57:48,951] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 83% 456/549 [46:03<07:10,  4.63s/it]
+
+
                                 
 83% 457/549 [46:07<12:27,  8.13s/it]
 83% 458/549 [46:11<10:23,  6.86s/it]
 84% 459/549 [46:15<08:56,  5.96s/it]
 84% 460/549 [46:19<07:55,  5.34s/it]
 84% 461/549 [46:23<07:11,  4.91s/it]
 84% 462/549 [46:26<06:40,  4.60s/it][2025-10-10 13:58:18,149] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:58:20,763] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2804484367370605
+[2025-10-10 13:58:22,047] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2839665412902832
+[2025-10-10 13:58:23,316] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2688114643096924
+[2025-10-10 13:58:24,612] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2959990501403809
+[2025-10-10 13:58:24,613] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                     
+
                                 
+
 84% 462/549 [46:39<06:40,  4.60s/it]
+
+
                                 
 84% 463/549 [46:43<11:33,  8.06s/it]
 85% 464/549 [46:46<09:37,  6.80s/it]
 85% 465/549 [46:50<08:16,  5.91s/it]
 85% 466/549 [46:54<07:19,  5.30s/it]
 85% 467/549 [46:58<06:39,  4.87s/it]
 85% 468/549 [47:02<06:11,  4.59s/it][2025-10-10 13:58:53,663] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:58:56,265] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2601747512817383
+[2025-10-10 13:58:57,546] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2804811000823975
+[2025-10-10 13:58:58,818] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2715575695037842
+[2025-10-10 13:59:00,085] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2669270038604736
+[2025-10-10 13:59:00,085] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 85% 468/549 [47:14<06:11,  4.59s/it]
+
+
                                 
 85% 469/549 [47:18<10:44,  8.05s/it]
 86% 470/549 [47:22<08:56,  6.80s/it]
 86% 471/549 [47:26<07:43,  5.94s/it]
 86% 472/549 [47:30<06:50,  5.33s/it]
 86% 473/549 [47:34<06:12,  4.90s/it]
 86% 474/549 [47:38<05:45,  4.61s/it][2025-10-10 13:59:29,330] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 13:59:32,429] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2759573459625244
+[2025-10-10 13:59:33,742] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3128514289855957
+[2025-10-10 13:59:35,045] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3031983375549316
+[2025-10-10 13:59:36,349] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3030741214752197
+[2025-10-10 13:59:36,349] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 86% 474/549 [47:51<05:45,  4.61s/it]
+
+
                                 
 87% 475/549 [47:54<10:11,  8.26s/it]
 87% 476/549 [47:58<08:26,  6.94s/it]
 87% 477/549 [48:02<07:13,  6.02s/it]
 87% 478/549 [48:06<06:21,  5.38s/it]
 87% 479/549 [48:10<05:45,  4.93s/it]
 87% 480/549 [48:14<05:19,  4.63s/it][2025-10-10 14:00:05,526] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:00:08,809] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4298784732818604
+[2025-10-10 14:00:10,095] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.285125732421875
+[2025-10-10 14:00:11,356] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2615094184875488
+[2025-10-10 14:00:12,652] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.294884443283081
+[2025-10-10 14:00:12,652] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 87% 480/549 [48:27<05:19,  4.63s/it]
+
+
                                 
 88% 481/549 [48:31<09:23,  8.28s/it]
 88% 482/549 [48:35<07:45,  6.95s/it]
 88% 483/549 [48:38<06:37,  6.03s/it]
 88% 484/549 [48:42<05:50,  5.39s/it]
 88% 485/549 [48:46<05:16,  4.95s/it]
 89% 486/549 [48:50<04:52,  4.64s/it][2025-10-10 14:00:41,802] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:00:45,094] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8106629848480225
+[2025-10-10 14:00:46,391] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2969591617584229
+[2025-10-10 14:00:47,689] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.297926425933838
+[2025-10-10 14:00:48,990] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3006622791290283
+[2025-10-10 14:00:48,991] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 89% 486/549 [49:03<04:52,  4.64s/it]
+
+
                                 
 89% 487/549 [49:07<08:35,  8.31s/it]
 89% 488/549 [49:11<07:05,  6.97s/it]
 89% 489/549 [49:15<06:02,  6.04s/it]
 89% 490/549 [49:19<05:18,  5.39s/it]
 89% 491/549 [49:23<04:46,  4.94s/it]
 90% 492/549 [49:26<04:23,  4.63s/it][2025-10-10 14:01:18,064] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:01:21,207] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.8038432598114014
+[2025-10-10 14:01:22,618] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4110236167907715
+[2025-10-10 14:01:23,883] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.264404296875
+[2025-10-10 14:01:25,172] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.288550615310669
+[2025-10-10 14:01:25,172] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 90% 492/549 [49:39<04:23,  4.63s/it]
+
+
                                 
 90% 493/549 [49:43<07:42,  8.26s/it]
 90% 494/549 [49:47<06:21,  6.94s/it]
 90% 495/549 [49:51<05:24,  6.01s/it]
 90% 496/549 [49:55<04:44,  5.37s/it]
 91% 497/549 [49:59<04:16,  4.93s/it]
 91% 498/549 [50:03<03:55,  4.62s/it][2025-10-10 14:01:54,168] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:01:56,896] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4492151737213135
+[2025-10-10 14:01:58,644] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7475955486297607
+[2025-10-10 14:01:59,912] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2675278186798096
+[2025-10-10 14:02:01,187] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2756171226501465
+[2025-10-10 14:02:01,188] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 91% 498/549 [50:15<03:55,  4.62s/it]
+
+
                                 
 91% 499/549 [50:19<06:51,  8.22s/it]
 91% 500/549 [50:23<05:38,  6.91s/it]
                                     
+
 91% 500/549 [50:23<05:38,  6.91s/it]
 91% 501/549 [50:27<04:47,  6.00s/it]
 91% 502/549 [50:31<04:12,  5.37s/it]
 92% 503/549 [50:35<03:46,  4.93s/it]
 92% 504/549 [50:39<03:28,  4.63s/it][2025-10-10 14:02:30,241] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:02:32,802] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2650606632232666
+[2025-10-10 14:02:34,502] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6998417377471924
+[2025-10-10 14:02:36,052] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5492708683013916
+[2025-10-10 14:02:37,338] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2862026691436768
+[2025-10-10 14:02:37,338] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                     
+
                                 
+
 92% 504/549 [50:52<03:28,  4.63s/it]
+
+
                                 
 92% 505/549 [50:55<06:04,  8.28s/it]
 92% 506/549 [50:59<04:59,  6.96s/it]
 92% 507/549 [51:03<04:13,  6.04s/it]
 93% 508/549 [51:07<03:41,  5.40s/it]
 93% 509/549 [51:11<03:17,  4.95s/it]
 93% 510/549 [51:15<03:00,  4.63s/it][2025-10-10 14:03:06,511] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:03:09,143] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3052573204040527
+[2025-10-10 14:03:10,598] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4548754692077637
+[2025-10-10 14:03:12,399] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.800527811050415
+[2025-10-10 14:03:13,696] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2964775562286377
+[2025-10-10 14:03:13,696] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 93% 510/549 [51:28<03:00,  4.63s/it]
+
+
                                 
 93% 511/549 [51:32<05:15,  8.31s/it]
 93% 512/549 [51:36<04:18,  6.98s/it]
 93% 513/549 [51:39<03:37,  6.04s/it]
 94% 514/549 [51:43<03:09,  5.40s/it]
 94% 515/549 [51:47<02:48,  4.95s/it]
 94% 516/549 [51:51<02:32,  4.64s/it][2025-10-10 14:03:42,833] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:03:45,384] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2671685218811035
+[2025-10-10 14:03:46,651] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2670958042144775
+[2025-10-10 14:03:48,344] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.6930501461029053
+[2025-10-10 14:03:49,888] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.5432307720184326
+[2025-10-10 14:03:49,888] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 94% 516/549 [52:04<02:32,  4.64s/it]
+
+
                                 
 94% 517/549 [52:08<04:25,  8.31s/it]
 94% 518/549 [52:12<03:36,  6.97s/it]
 95% 519/549 [52:16<03:01,  6.04s/it]
 95% 520/549 [52:20<02:36,  5.40s/it]
 95% 521/549 [52:24<02:18,  4.95s/it]
 95% 522/549 [52:27<02:04,  4.63s/it][2025-10-10 14:04:19,117] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:04:21,739] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.3012058734893799
+[2025-10-10 14:04:23,021] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2816236019134521
+[2025-10-10 14:04:24,451] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.4290544986724854
+[2025-10-10 14:04:26,225] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.7740297317504883
+[2025-10-10 14:04:26,225] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 95% 522/549 [52:40<02:04,  4.63s/it]
+
+
                                 
 95% 523/549 [52:44<03:34,  8.26s/it]
 95% 524/549 [52:48<02:53,  6.94s/it]
 96% 525/549 [52:52<02:24,  6.01s/it]
 96% 526/549 [52:56<02:03,  5.37s/it]
 96% 527/549 [53:00<01:48,  4.93s/it]
 96% 528/549 [53:04<01:37,  4.62s/it][2025-10-10 14:04:55,239] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:04:57,810] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2706298828125
+[2025-10-10 14:04:59,088] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.276904582977295
+[2025-10-10 14:05:00,384] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2962520122528076
+[2025-10-10 14:05:02,135] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.750046730041504
+[2025-10-10 14:05:02,135] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.84it/s]
+
 75% 3/4 [00:02<00:00,  1.32it/s]
+
                                     
+
                                 
+
 96% 528/549 [53:16<01:37,  4.62s/it]
+
+
                                 
 96% 529/549 [53:20<02:45,  8.26s/it]
 97% 530/549 [53:24<02:11,  6.95s/it]
 97% 531/549 [53:28<01:48,  6.03s/it]
 97% 532/549 [53:32<01:31,  5.39s/it]
 97% 533/549 [53:36<01:19,  4.95s/it]
 97% 534/549 [53:40<01:09,  4.63s/it][2025-10-10 14:05:31,473] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:05:34,055] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.281491994857788
+[2025-10-10 14:05:35,343] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2869987487792969
+[2025-10-10 14:05:36,619] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.275632381439209
+[2025-10-10 14:05:38,079] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.460437297821045
+[2025-10-10 14:05:38,080] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 97% 534/549 [53:53<01:09,  4.63s/it]
+
+
                                 
 97% 535/549 [53:57<01:55,  8.26s/it]
 98% 536/549 [54:00<01:30,  6.93s/it]
 98% 537/549 [54:04<01:12,  6.01s/it]
 98% 538/549 [54:08<00:59,  5.37s/it]
 98% 539/549 [54:12<00:49,  4.93s/it]
 98% 540/549 [54:16<00:41,  4.62s/it][2025-10-10 14:06:07,574] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:06:10,143] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2767753601074219
+[2025-10-10 14:06:11,428] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2852528095245361
+[2025-10-10 14:06:12,701] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2729084491729736
+[2025-10-10 14:06:13,962] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2607090473175049
+[2025-10-10 14:06:13,963] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 98% 540/549 [54:29<00:41,  4.62s/it]
+
+
                                 
 99% 541/549 [54:32<01:05,  8.20s/it]
 99% 542/549 [54:36<00:48,  6.91s/it]
 99% 543/549 [54:40<00:36,  6.00s/it]
 99% 544/549 [54:44<00:26,  5.37s/it]
 99% 545/549 [54:48<00:19,  4.94s/it]
 99% 546/549 [54:52<00:13,  4.63s/it][2025-10-10 14:06:43,637] [INFO] [axolotl.core.trainers.base.evaluate:376] [PID:24741] Running evaluation step...
+[2025-10-10 14:06:46,208] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2853760719299316
+[2025-10-10 14:06:47,466] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2572674751281738
+[2025-10-10 14:06:48,724] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2585771083831787
+[2025-10-10 14:06:50,005] [DEBUG] [axolotl.utils.samplers.multipack.__len__:462] [PID:24741] generate_batches time: 1.2800981998443604
+[2025-10-10 14:06:50,005] [INFO] [axolotl.utils.samplers.multipack.calc_min_len:438] [PID:24741] gather_len_batches: [4]
+
+
  0% 0/4 [00:00<?, ?it/s]
+
 50% 2/4 [00:01<00:01,  1.85it/s]
+
 75% 3/4 [00:02<00:00,  1.33it/s]
+
                                     
+
                                 
+
 99% 546/549 [55:04<00:13,  4.63s/it]
+
+
                                 
+
                                     
+