Training in progress, step 40000

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36d434c8d0a711df4d3800a7e154a8fbaa4770ef11a77b22a1edbefe79747963
+oid sha256:ce0a6b5f6498d58bc71192d05ec18b2d0e8aa62a4e47f97c5dff64f06a69cc83
 size 460928

trainer_log.jsonl

@@ -8157,3 +8157,45 @@
 {"current_steps": 39795, "total_steps": 40000, "loss": 0.0, "lr": 1.9632065447422463e-05, "epoch": 193.17917675544794, "percentage": 99.49, "elapsed_time": "4:45:24", "remaining_time": "0:01:28", "throughput": 3982.31, "total_tokens": 68194592}
 {"current_steps": 39800, "total_steps": 40000, "loss": 0.0, "lr": 1.8690637803880916e-05, "epoch": 193.20338983050848, "percentage": 99.5, "elapsed_time": "4:45:26", "remaining_time": "0:01:26", "throughput": 3982.35, "total_tokens": 68203104}
 {"current_steps": 39800, "total_steps": 40000, "eval_loss": 1.1948976516723633, "epoch": 193.20338983050848, "percentage": 99.5, "elapsed_time": "4:45:30", "remaining_time": "0:01:26", "throughput": 3981.27, "total_tokens": 68203104}
+{"current_steps": 39805, "total_steps": 40000, "loss": 0.0, "lr": 1.7772339163019123e-05, "epoch": 193.227602905569, "percentage": 99.51, "elapsed_time": "4:45:34", "remaining_time": "0:01:23", "throughput": 3980.97, "total_tokens": 68211744}
+{"current_steps": 39810, "total_steps": 40000, "loss": 0.0, "lr": 1.6877169666457138e-05, "epoch": 193.25181598062954, "percentage": 99.52, "elapsed_time": "4:45:36", "remaining_time": "0:01:21", "throughput": 3981.0, "total_tokens": 68220192}
+{"current_steps": 39815, "total_steps": 40000, "loss": 0.0, "lr": 1.6005129452234532e-05, "epoch": 193.27602905569006, "percentage": 99.54, "elapsed_time": "4:45:38", "remaining_time": "0:01:19", "throughput": 3981.03, "total_tokens": 68228576}
+{"current_steps": 39820, "total_steps": 40000, "loss": 0.0, "lr": 1.5156218654843733e-05, "epoch": 193.3002421307506, "percentage": 99.55, "elapsed_time": "4:45:40", "remaining_time": "0:01:17", "throughput": 3981.05, "total_tokens": 68236768}
+{"current_steps": 39825, "total_steps": 40000, "loss": 0.0, "lr": 1.4330437405196683e-05, "epoch": 193.32445520581115, "percentage": 99.56, "elapsed_time": "4:45:42", "remaining_time": "0:01:15", "throughput": 3981.08, "total_tokens": 68245216}
+{"current_steps": 39830, "total_steps": 40000, "loss": 0.0, "lr": 1.352778583062486e-05, "epoch": 193.34866828087166, "percentage": 99.58, "elapsed_time": "4:45:44", "remaining_time": "0:01:13", "throughput": 3981.12, "total_tokens": 68253696}
+{"current_steps": 39835, "total_steps": 40000, "loss": 0.0, "lr": 1.2748264054929237e-05, "epoch": 193.3728813559322, "percentage": 99.59, "elapsed_time": "4:45:46", "remaining_time": "0:01:11", "throughput": 3981.18, "total_tokens": 68262624}
+{"current_steps": 39840, "total_steps": 40000, "loss": 0.0, "lr": 1.1991872198297004e-05, "epoch": 193.39709443099272, "percentage": 99.6, "elapsed_time": "4:45:48", "remaining_time": "0:01:08", "throughput": 3981.2, "total_tokens": 68270976}
+{"current_steps": 39845, "total_steps": 40000, "loss": 0.0, "lr": 1.1258610377384847e-05, "epoch": 193.42130750605327, "percentage": 99.61, "elapsed_time": "4:45:50", "remaining_time": "0:01:06", "throughput": 3981.24, "total_tokens": 68279360}
+{"current_steps": 39850, "total_steps": 40000, "loss": 0.0, "lr": 1.0548478705268982e-05, "epoch": 193.4455205811138, "percentage": 99.62, "elapsed_time": "4:45:52", "remaining_time": "0:01:04", "throughput": 3981.26, "total_tokens": 68287712}
+{"current_steps": 39855, "total_steps": 40000, "loss": 0.0, "lr": 9.86147729147846e-06, "epoch": 193.46973365617433, "percentage": 99.64, "elapsed_time": "4:45:54", "remaining_time": "0:01:02", "throughput": 3981.29, "total_tokens": 68295968}
+{"current_steps": 39860, "total_steps": 40000, "loss": 0.0, "lr": 9.197606241928557e-06, "epoch": 193.49394673123487, "percentage": 99.65, "elapsed_time": "4:45:56", "remaining_time": "0:01:00", "throughput": 3981.34, "total_tokens": 68304800}
+{"current_steps": 39865, "total_steps": 40000, "loss": 0.0, "lr": 8.556865659004042e-06, "epoch": 193.5181598062954, "percentage": 99.66, "elapsed_time": "4:45:58", "remaining_time": "0:00:58", "throughput": 3981.39, "total_tokens": 68313472}
+{"current_steps": 39870, "total_steps": 40000, "loss": 0.0, "lr": 7.939255641525867e-06, "epoch": 193.54237288135593, "percentage": 99.67, "elapsed_time": "4:46:00", "remaining_time": "0:00:55", "throughput": 3981.41, "total_tokens": 68321792}
+{"current_steps": 39875, "total_steps": 40000, "loss": 0.0, "lr": 7.344776284751164e-06, "epoch": 193.56658595641647, "percentage": 99.69, "elapsed_time": "4:46:02", "remaining_time": "0:00:53", "throughput": 3981.46, "total_tokens": 68330656}
+{"current_steps": 39880, "total_steps": 40000, "loss": 0.0, "lr": 6.773427680323296e-06, "epoch": 193.590799031477, "percentage": 99.7, "elapsed_time": "4:46:04", "remaining_time": "0:00:51", "throughput": 3981.5, "total_tokens": 68339232}
+{"current_steps": 39885, "total_steps": 40000, "loss": 0.0, "lr": 6.225209916355112e-06, "epoch": 193.61501210653753, "percentage": 99.71, "elapsed_time": "4:46:06", "remaining_time": "0:00:49", "throughput": 3981.54, "total_tokens": 68347744}
+{"current_steps": 39890, "total_steps": 40000, "loss": 0.0, "lr": 5.7001230774123e-06, "epoch": 193.63922518159805, "percentage": 99.72, "elapsed_time": "4:46:08", "remaining_time": "0:00:47", "throughput": 3981.57, "total_tokens": 68356192}
+{"current_steps": 39895, "total_steps": 40000, "loss": 0.0, "lr": 5.198167244446772e-06, "epoch": 193.6634382566586, "percentage": 99.74, "elapsed_time": "4:46:10", "remaining_time": "0:00:45", "throughput": 3981.61, "total_tokens": 68364768}
+{"current_steps": 39900, "total_steps": 40000, "loss": 0.0, "lr": 4.71934249487993e-06, "epoch": 193.68765133171914, "percentage": 99.75, "elapsed_time": "4:46:12", "remaining_time": "0:00:43", "throughput": 3981.66, "total_tokens": 68373568}
+{"current_steps": 39905, "total_steps": 40000, "loss": 0.0, "lr": 4.2636489025527075e-06, "epoch": 193.71186440677965, "percentage": 99.76, "elapsed_time": "4:46:14", "remaining_time": "0:00:40", "throughput": 3981.71, "total_tokens": 68382272}
+{"current_steps": 39910, "total_steps": 40000, "loss": 0.0, "lr": 3.831086537742223e-06, "epoch": 193.7360774818402, "percentage": 99.78, "elapsed_time": "4:46:16", "remaining_time": "0:00:38", "throughput": 3981.75, "total_tokens": 68390784}
+{"current_steps": 39915, "total_steps": 40000, "loss": 0.0, "lr": 3.4216554671451236e-06, "epoch": 193.7602905569007, "percentage": 99.79, "elapsed_time": "4:46:18", "remaining_time": "0:00:36", "throughput": 3981.77, "total_tokens": 68399008}
+{"current_steps": 39920, "total_steps": 40000, "loss": 0.0, "lr": 3.035355753894242e-06, "epoch": 193.78450363196126, "percentage": 99.8, "elapsed_time": "4:46:20", "remaining_time": "0:00:34", "throughput": 3981.8, "total_tokens": 68407520}
+{"current_steps": 39925, "total_steps": 40000, "loss": 0.0, "lr": 2.6721874575752477e-06, "epoch": 193.8087167070218, "percentage": 99.81, "elapsed_time": "4:46:22", "remaining_time": "0:00:32", "throughput": 3981.85, "total_tokens": 68416192}
+{"current_steps": 39930, "total_steps": 40000, "loss": 0.0, "lr": 2.3321506341933418e-06, "epoch": 193.83292978208232, "percentage": 99.83, "elapsed_time": "4:46:24", "remaining_time": "0:00:30", "throughput": 3981.9, "total_tokens": 68425056}
+{"current_steps": 39935, "total_steps": 40000, "loss": 0.0, "lr": 2.0152453361732546e-06, "epoch": 193.85714285714286, "percentage": 99.84, "elapsed_time": "4:46:25", "remaining_time": "0:00:27", "throughput": 3981.96, "total_tokens": 68433856}
+{"current_steps": 39940, "total_steps": 40000, "loss": 0.0, "lr": 1.7214716123925554e-06, "epoch": 193.88135593220338, "percentage": 99.85, "elapsed_time": "4:46:27", "remaining_time": "0:00:25", "throughput": 3981.97, "total_tokens": 68441984}
+{"current_steps": 39945, "total_steps": 40000, "loss": 0.0, "lr": 1.4508295081649968e-06, "epoch": 193.90556900726392, "percentage": 99.86, "elapsed_time": "4:46:29", "remaining_time": "0:00:23", "throughput": 3982.03, "total_tokens": 68450912}
+{"current_steps": 39950, "total_steps": 40000, "loss": 0.0, "lr": 1.2033190652238623e-06, "epoch": 193.92978208232446, "percentage": 99.88, "elapsed_time": "4:46:31", "remaining_time": "0:00:21", "throughput": 3982.06, "total_tokens": 68459296}
+{"current_steps": 39955, "total_steps": 40000, "loss": 0.0, "lr": 9.78940321721966e-07, "epoch": 193.95399515738498, "percentage": 99.89, "elapsed_time": "4:46:33", "remaining_time": "0:00:19", "throughput": 3982.09, "total_tokens": 68467744}
+{"current_steps": 39960, "total_steps": 40000, "loss": 0.0, "lr": 7.776933122816132e-07, "epoch": 193.97820823244552, "percentage": 99.9, "elapsed_time": "4:46:35", "remaining_time": "0:00:17", "throughput": 3982.13, "total_tokens": 68476256}
+{"current_steps": 39965, "total_steps": 40000, "loss": 0.0, "lr": 5.99578067927986e-07, "epoch": 194.0048426150121, "percentage": 99.91, "elapsed_time": "4:46:38", "remaining_time": "0:00:15", "throughput": 3982.09, "total_tokens": 68485248}
+{"current_steps": 39970, "total_steps": 40000, "loss": 0.0, "lr": 4.445946161224512e-07, "epoch": 194.02905569007265, "percentage": 99.92, "elapsed_time": "4:46:40", "remaining_time": "0:00:12", "throughput": 3982.12, "total_tokens": 68493536}
+{"current_steps": 39975, "total_steps": 40000, "loss": 0.0, "lr": 3.127429807792126e-07, "epoch": 194.05326876513317, "percentage": 99.94, "elapsed_time": "4:46:42", "remaining_time": "0:00:10", "throughput": 3982.17, "total_tokens": 68502304}
+{"current_steps": 39980, "total_steps": 40000, "loss": 0.0, "lr": 2.040231822320049e-07, "epoch": 194.0774818401937, "percentage": 99.95, "elapsed_time": "4:46:44", "remaining_time": "0:00:08", "throughput": 3982.15, "total_tokens": 68510848}
+{"current_steps": 39985, "total_steps": 40000, "loss": 0.0, "lr": 1.1843523723409354e-07, "epoch": 194.10169491525423, "percentage": 99.96, "elapsed_time": "4:46:46", "remaining_time": "0:00:06", "throughput": 3982.19, "total_tokens": 68519488}
+{"current_steps": 39990, "total_steps": 40000, "loss": 0.0, "lr": 5.597915897492811e-08, "epoch": 194.12590799031477, "percentage": 99.98, "elapsed_time": "4:46:48", "remaining_time": "0:00:04", "throughput": 3982.21, "total_tokens": 68527808}
+{"current_steps": 39995, "total_steps": 40000, "loss": 0.0, "lr": 1.6654957113448885e-08, "epoch": 194.15012106537532, "percentage": 99.99, "elapsed_time": "4:46:50", "remaining_time": "0:00:02", "throughput": 3982.25, "total_tokens": 68536288}
+{"current_steps": 40000, "total_steps": 40000, "loss": 0.0, "lr": 4.626377114735902e-10, "epoch": 194.17433414043583, "percentage": 100.0, "elapsed_time": "4:46:52", "remaining_time": "0:00:00", "throughput": 3982.29, "total_tokens": 68544800}
+{"current_steps": 40000, "total_steps": 40000, "eval_loss": 1.1948976516723633, "epoch": 194.17433414043583, "percentage": 100.0, "elapsed_time": "4:46:57", "remaining_time": "0:00:00", "throughput": 3981.22, "total_tokens": 68544800}
+{"current_steps": 40000, "total_steps": 40000, "epoch": 194.17433414043583, "percentage": 100.0, "elapsed_time": "4:46:58", "remaining_time": "0:00:00", "throughput": 3980.89, "total_tokens": 68544800}