Training in progress, step 39800

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8321069730f2c186dee149f9bd83e38dbb0128a97d9d3de1f648e0e7841cdd32
+oid sha256:6f10c2a90fe351d5fbf00718f650e4713ec97d9146be7361bb93ed97c6244a84
 size 1638528

trainer_log.jsonl

@@ -8117,3 +8117,43 @@
 {"current_steps": 39600, "total_steps": 40000, "loss": 0.0, "lr": 7.4386456836667e-05, "epoch": 694.7433628318585, "percentage": 99.0, "elapsed_time": "2:52:33", "remaining_time": "0:01:44", "throughput": 2119.35, "total_tokens": 21942280}
 {"current_steps": 39600, "total_steps": 40000, "eval_loss": 0.6724047064781189, "epoch": 694.7433628318585, "percentage": 99.0, "elapsed_time": "2:52:34", "remaining_time": "0:01:44", "throughput": 2119.12, "total_tokens": 21942280}
 {"current_steps": 39605, "total_steps": 40000, "loss": 0.0, "lr": 7.254314656586214e-05, "epoch": 694.8318584070796, "percentage": 99.01, "elapsed_time": "2:52:36", "remaining_time": "0:01:43", "throughput": 2118.93, "total_tokens": 21945032}
+{"current_steps": 39610, "total_steps": 40000, "loss": 0.0, "lr": 7.07229569929968e-05, "epoch": 694.9203539823009, "percentage": 99.02, "elapsed_time": "2:52:37", "remaining_time": "0:01:41", "throughput": 2118.95, "total_tokens": 21947880}
+{"current_steps": 39615, "total_steps": 40000, "loss": 0.0, "lr": 6.892588839879643e-05, "epoch": 695.0, "percentage": 99.04, "elapsed_time": "2:52:39", "remaining_time": "0:01:40", "throughput": 2118.95, "total_tokens": 21950592}
+{"current_steps": 39620, "total_steps": 40000, "loss": 0.0, "lr": 6.71519410603727e-05, "epoch": 695.0884955752213, "percentage": 99.05, "elapsed_time": "2:52:40", "remaining_time": "0:01:39", "throughput": 2118.92, "total_tokens": 21953456}
+{"current_steps": 39625, "total_steps": 40000, "loss": 0.0, "lr": 6.540111525129011e-05, "epoch": 695.1769911504425, "percentage": 99.06, "elapsed_time": "2:52:41", "remaining_time": "0:01:38", "throughput": 2118.95, "total_tokens": 21956432}
+{"current_steps": 39630, "total_steps": 40000, "loss": 0.0, "lr": 6.367341124154934e-05, "epoch": 695.2654867256637, "percentage": 99.08, "elapsed_time": "2:52:43", "remaining_time": "0:01:36", "throughput": 2118.98, "total_tokens": 21959472}
+{"current_steps": 39635, "total_steps": 40000, "loss": 0.0, "lr": 6.19688292975873e-05, "epoch": 695.3539823008849, "percentage": 99.09, "elapsed_time": "2:52:44", "remaining_time": "0:01:35", "throughput": 2118.96, "total_tokens": 21961632}
+{"current_steps": 39640, "total_steps": 40000, "loss": 0.0, "lr": 6.0287369682260336e-05, "epoch": 695.4424778761062, "percentage": 99.1, "elapsed_time": "2:52:45", "remaining_time": "0:01:34", "throughput": 2118.96, "total_tokens": 21964240}
+{"current_steps": 39645, "total_steps": 40000, "loss": 0.0, "lr": 5.8629032654894384e-05, "epoch": 695.5309734513274, "percentage": 99.11, "elapsed_time": "2:52:46", "remaining_time": "0:01:32", "throughput": 2118.99, "total_tokens": 21967120}
+{"current_steps": 39650, "total_steps": 40000, "loss": 0.0, "lr": 5.699381847120155e-05, "epoch": 695.6194690265487, "percentage": 99.12, "elapsed_time": "2:52:48", "remaining_time": "0:01:31", "throughput": 2119.0, "total_tokens": 21969936}
+{"current_steps": 39655, "total_steps": 40000, "loss": 0.0, "lr": 5.5381727383380094e-05, "epoch": 695.70796460177, "percentage": 99.14, "elapsed_time": "2:52:49", "remaining_time": "0:01:30", "throughput": 2119.01, "total_tokens": 21972608}
+{"current_steps": 39660, "total_steps": 40000, "loss": 0.0, "lr": 5.379275964001451e-05, "epoch": 695.7964601769911, "percentage": 99.15, "elapsed_time": "2:52:50", "remaining_time": "0:01:28", "throughput": 2119.01, "total_tokens": 21975232}
+{"current_steps": 39665, "total_steps": 40000, "loss": 0.0, "lr": 5.222691548614211e-05, "epoch": 695.8849557522124, "percentage": 99.16, "elapsed_time": "2:52:51", "remaining_time": "0:01:27", "throughput": 2119.06, "total_tokens": 21978400}
+{"current_steps": 39670, "total_steps": 40000, "loss": 0.0, "lr": 5.068419516323641e-05, "epoch": 695.9734513274336, "percentage": 99.17, "elapsed_time": "2:52:53", "remaining_time": "0:01:26", "throughput": 2119.11, "total_tokens": 21981664}
+{"current_steps": 39675, "total_steps": 40000, "loss": 0.0, "lr": 4.91645989092071e-05, "epoch": 696.0530973451328, "percentage": 99.19, "elapsed_time": "2:52:54", "remaining_time": "0:01:24", "throughput": 2119.03, "total_tokens": 21983856}
+{"current_steps": 39680, "total_steps": 40000, "loss": 0.0, "lr": 4.7668126958400056e-05, "epoch": 696.141592920354, "percentage": 99.2, "elapsed_time": "2:52:55", "remaining_time": "0:01:23", "throughput": 2119.09, "total_tokens": 21987232}
+{"current_steps": 39685, "total_steps": 40000, "loss": 0.0, "lr": 4.619477954159734e-05, "epoch": 696.2300884955753, "percentage": 99.21, "elapsed_time": "2:52:56", "remaining_time": "0:01:22", "throughput": 2119.06, "total_tokens": 21989440}
+{"current_steps": 39690, "total_steps": 40000, "loss": 0.0, "lr": 4.4744556885983884e-05, "epoch": 696.3185840707964, "percentage": 99.22, "elapsed_time": "2:52:58", "remaining_time": "0:01:21", "throughput": 2119.12, "total_tokens": 21992976}
+{"current_steps": 39695, "total_steps": 40000, "loss": 0.0, "lr": 4.331745921523078e-05, "epoch": 696.4070796460177, "percentage": 99.24, "elapsed_time": "2:52:59", "remaining_time": "0:01:19", "throughput": 2119.14, "total_tokens": 21995776}
+{"current_steps": 39700, "total_steps": 40000, "loss": 0.0, "lr": 4.191348674937867e-05, "epoch": 696.4955752212389, "percentage": 99.25, "elapsed_time": "2:53:00", "remaining_time": "0:01:18", "throughput": 2119.15, "total_tokens": 21998432}
+{"current_steps": 39705, "total_steps": 40000, "loss": 0.0, "lr": 4.0532639704971006e-05, "epoch": 696.5840707964602, "percentage": 99.26, "elapsed_time": "2:53:01", "remaining_time": "0:01:17", "throughput": 2119.16, "total_tokens": 22001072}
+{"current_steps": 39710, "total_steps": 40000, "loss": 0.0, "lr": 3.917491829493747e-05, "epoch": 696.6725663716815, "percentage": 99.28, "elapsed_time": "2:53:03", "remaining_time": "0:01:15", "throughput": 2119.16, "total_tokens": 22003616}
+{"current_steps": 39715, "total_steps": 40000, "loss": 0.0, "lr": 3.78403227286439e-05, "epoch": 696.7610619469026, "percentage": 99.29, "elapsed_time": "2:53:04", "remaining_time": "0:01:14", "throughput": 2119.18, "total_tokens": 22006480}
+{"current_steps": 39720, "total_steps": 40000, "loss": 0.0, "lr": 3.652885321192567e-05, "epoch": 696.8495575221239, "percentage": 99.3, "elapsed_time": "2:53:05", "remaining_time": "0:01:13", "throughput": 2119.22, "total_tokens": 22009520}
+{"current_steps": 39725, "total_steps": 40000, "loss": 0.0, "lr": 3.524050994702099e-05, "epoch": 696.9380530973451, "percentage": 99.31, "elapsed_time": "2:53:06", "remaining_time": "0:01:11", "throughput": 2119.23, "total_tokens": 22012224}
+{"current_steps": 39730, "total_steps": 40000, "loss": 0.0, "lr": 3.3975293132604276e-05, "epoch": 697.0176991150443, "percentage": 99.33, "elapsed_time": "2:53:08", "remaining_time": "0:01:10", "throughput": 2119.14, "total_tokens": 22014416}
+{"current_steps": 39735, "total_steps": 40000, "loss": 0.0, "lr": 3.2733202963786125e-05, "epoch": 697.1061946902655, "percentage": 99.34, "elapsed_time": "2:53:09", "remaining_time": "0:01:09", "throughput": 2119.18, "total_tokens": 22017440}
+{"current_steps": 39740, "total_steps": 40000, "loss": 0.0, "lr": 3.15142396321133e-05, "epoch": 697.1946902654868, "percentage": 99.35, "elapsed_time": "2:53:10", "remaining_time": "0:01:07", "throughput": 2119.21, "total_tokens": 22020464}
+{"current_steps": 39745, "total_steps": 40000, "loss": 0.0, "lr": 3.0318403325552132e-05, "epoch": 697.2831858407079, "percentage": 99.36, "elapsed_time": "2:53:12", "remaining_time": "0:01:06", "throughput": 2119.22, "total_tokens": 22023232}
+{"current_steps": 39750, "total_steps": 40000, "loss": 0.0, "lr": 2.914569422855506e-05, "epoch": 697.3716814159292, "percentage": 99.38, "elapsed_time": "2:53:13", "remaining_time": "0:01:05", "throughput": 2119.24, "total_tokens": 22026016}
+{"current_steps": 39755, "total_steps": 40000, "loss": 0.0, "lr": 2.7996112521927462e-05, "epoch": 697.4601769911504, "percentage": 99.39, "elapsed_time": "2:53:14", "remaining_time": "0:01:04", "throughput": 2119.26, "total_tokens": 22028704}
+{"current_steps": 39760, "total_steps": 40000, "loss": 0.0, "lr": 2.68696583829775e-05, "epoch": 697.5486725663717, "percentage": 99.4, "elapsed_time": "2:53:15", "remaining_time": "0:01:02", "throughput": 2119.25, "total_tokens": 22031088}
+{"current_steps": 39765, "total_steps": 40000, "loss": 0.0, "lr": 2.576633198539957e-05, "epoch": 697.637168141593, "percentage": 99.41, "elapsed_time": "2:53:16", "remaining_time": "0:01:01", "throughput": 2119.27, "total_tokens": 22033792}
+{"current_steps": 39770, "total_steps": 40000, "loss": 0.0, "lr": 2.46861334993409e-05, "epoch": 697.7256637168142, "percentage": 99.42, "elapsed_time": "2:53:18", "remaining_time": "0:01:00", "throughput": 2119.31, "total_tokens": 22036928}
+{"current_steps": 39775, "total_steps": 40000, "loss": 0.0, "lr": 2.3629063091384903e-05, "epoch": 697.8141592920354, "percentage": 99.44, "elapsed_time": "2:53:19", "remaining_time": "0:00:58", "throughput": 2119.33, "total_tokens": 22039776}
+{"current_steps": 39780, "total_steps": 40000, "loss": 0.0, "lr": 2.2595120924567834e-05, "epoch": 697.9026548672566, "percentage": 99.45, "elapsed_time": "2:53:20", "remaining_time": "0:00:57", "throughput": 2119.34, "total_tokens": 22042432}
+{"current_steps": 39785, "total_steps": 40000, "loss": 0.0, "lr": 2.158430715829551e-05, "epoch": 697.9911504424779, "percentage": 99.46, "elapsed_time": "2:53:21", "remaining_time": "0:00:56", "throughput": 2119.37, "total_tokens": 22045424}
+{"current_steps": 39790, "total_steps": 40000, "loss": 0.0, "lr": 2.059662194849321e-05, "epoch": 698.070796460177, "percentage": 99.48, "elapsed_time": "2:53:23", "remaining_time": "0:00:54", "throughput": 2119.28, "total_tokens": 22047560}
+{"current_steps": 39795, "total_steps": 40000, "loss": 0.0, "lr": 1.9632065447422463e-05, "epoch": 698.1592920353983, "percentage": 99.49, "elapsed_time": "2:53:24", "remaining_time": "0:00:53", "throughput": 2119.28, "total_tokens": 22050120}
+{"current_steps": 39800, "total_steps": 40000, "loss": 0.0, "lr": 1.8690637803880916e-05, "epoch": 698.2477876106195, "percentage": 99.5, "elapsed_time": "2:53:25", "remaining_time": "0:00:52", "throughput": 2119.32, "total_tokens": 22053128}
+{"current_steps": 39800, "total_steps": 40000, "eval_loss": 0.6596654653549194, "epoch": 698.2477876106195, "percentage": 99.5, "elapsed_time": "2:53:26", "remaining_time": "0:00:52", "throughput": 2119.09, "total_tokens": 22053128}