Training in progress, step 39800

trainer_log.jsonl

@@ -8116,3 +8116,44 @@
 {"current_steps": 39595, "total_steps": 40000, "loss": 0.6706, "lr": 1.2708814586862016e-08, "epoch": 1.2673644452979964, "percentage": 98.99, "elapsed_time": "1 day, 12:27:05", "remaining_time": "0:22:22", "throughput": 459.19, "total_tokens": 60257904}
 {"current_steps": 39600, "total_steps": 40000, "loss": 0.3274, "lr": 1.2397742806111168e-08, "epoch": 1.2675244862684847, "percentage": 99.0, "elapsed_time": "1 day, 12:27:07", "remaining_time": "0:22:05", "throughput": 459.24, "total_tokens": 60265552}
 {"current_steps": 39600, "total_steps": 40000, "eval_loss": 0.4941261410713196, "epoch": 1.2675244862684847, "percentage": 99.0, "elapsed_time": "1 day, 12:36:00", "remaining_time": "0:22:10", "throughput": 457.39, "total_tokens": 60265552}
+{"current_steps": 39605, "total_steps": 40000, "loss": 0.3374, "lr": 1.209052442764369e-08, "epoch": 1.2676845272389732, "percentage": 99.01, "elapsed_time": "1 day, 12:36:04", "remaining_time": "0:21:54", "throughput": 457.43, "total_tokens": 60273408}
+{"current_steps": 39610, "total_steps": 40000, "loss": 0.5062, "lr": 1.17871594988328e-08, "epoch": 1.2678445682094617, "percentage": 99.02, "elapsed_time": "1 day, 12:36:06", "remaining_time": "0:21:37", "throughput": 457.48, "total_tokens": 60280784}
+{"current_steps": 39615, "total_steps": 40000, "loss": 0.3611, "lr": 1.1487648066466072e-08, "epoch": 1.26800460917995, "percentage": 99.04, "elapsed_time": "1 day, 12:36:08", "remaining_time": "0:21:20", "throughput": 457.53, "total_tokens": 60288240}
+{"current_steps": 39620, "total_steps": 40000, "loss": 0.6453, "lr": 1.1191990176728784e-08, "epoch": 1.2681646501504384, "percentage": 99.05, "elapsed_time": "1 day, 12:36:11", "remaining_time": "0:21:03", "throughput": 457.58, "total_tokens": 60295504}
+{"current_steps": 39625, "total_steps": 40000, "loss": 0.437, "lr": 1.0900185875215018e-08, "epoch": 1.268324691120927, "percentage": 99.06, "elapsed_time": "1 day, 12:36:13", "remaining_time": "0:20:47", "throughput": 457.63, "total_tokens": 60302944}
+{"current_steps": 39630, "total_steps": 40000, "loss": 0.2737, "lr": 1.0612235206924891e-08, "epoch": 1.2684847320914154, "percentage": 99.08, "elapsed_time": "1 day, 12:36:16", "remaining_time": "0:20:30", "throughput": 457.67, "total_tokens": 60310416}
+{"current_steps": 39635, "total_steps": 40000, "loss": 0.3417, "lr": 1.0328138216264549e-08, "epoch": 1.268644773061904, "percentage": 99.09, "elapsed_time": "1 day, 12:36:18", "remaining_time": "0:20:13", "throughput": 457.72, "total_tokens": 60317984}
+{"current_steps": 39640, "total_steps": 40000, "loss": 0.5801, "lr": 1.004789494704339e-08, "epoch": 1.2688048140323924, "percentage": 99.1, "elapsed_time": "1 day, 12:36:21", "remaining_time": "0:19:56", "throughput": 457.77, "total_tokens": 60325600}
+{"current_steps": 39645, "total_steps": 40000, "loss": 0.6833, "lr": 9.771505442482397e-09, "epoch": 1.2689648550028807, "percentage": 99.11, "elapsed_time": "1 day, 12:36:23", "remaining_time": "0:19:40", "throughput": 457.82, "total_tokens": 60332912}
+{"current_steps": 39650, "total_steps": 40000, "loss": 0.5499, "lr": 9.498969745200259e-09, "epoch": 1.2691248959733692, "percentage": 99.12, "elapsed_time": "1 day, 12:36:26", "remaining_time": "0:19:23", "throughput": 457.87, "total_tokens": 60341056}
+{"current_steps": 39655, "total_steps": 40000, "loss": 0.4715, "lr": 9.230287897230017e-09, "epoch": 1.2692849369438577, "percentage": 99.14, "elapsed_time": "1 day, 12:36:28", "remaining_time": "0:19:06", "throughput": 457.92, "total_tokens": 60348512}
+{"current_steps": 39660, "total_steps": 40000, "loss": 0.4259, "lr": 8.965459940002419e-09, "epoch": 1.269444977914346, "percentage": 99.15, "elapsed_time": "1 day, 12:36:31", "remaining_time": "0:18:49", "throughput": 457.97, "total_tokens": 60356080}
+{"current_steps": 39665, "total_steps": 40000, "loss": 0.5032, "lr": 8.704485914357019e-09, "epoch": 1.2696050188848345, "percentage": 99.16, "elapsed_time": "1 day, 12:36:33", "remaining_time": "0:18:33", "throughput": 458.02, "total_tokens": 60363552}
+{"current_steps": 39670, "total_steps": 40000, "loss": 0.397, "lr": 8.447365860539402e-09, "epoch": 1.269765059855323, "percentage": 99.17, "elapsed_time": "1 day, 12:36:36", "remaining_time": "0:18:16", "throughput": 458.07, "total_tokens": 60371280}
+{"current_steps": 39675, "total_steps": 40000, "loss": 0.5856, "lr": 8.194099818201184e-09, "epoch": 1.2699251008258114, "percentage": 99.19, "elapsed_time": "1 day, 12:36:38", "remaining_time": "0:17:59", "throughput": 458.12, "total_tokens": 60379040}
+{"current_steps": 39680, "total_steps": 40000, "loss": 0.6218, "lr": 7.944687826400011e-09, "epoch": 1.2700851417963, "percentage": 99.2, "elapsed_time": "1 day, 12:36:40", "remaining_time": "0:17:42", "throughput": 458.16, "total_tokens": 60386528}
+{"current_steps": 39685, "total_steps": 40000, "loss": 0.4863, "lr": 7.699129923599557e-09, "epoch": 1.2702451827667882, "percentage": 99.21, "elapsed_time": "1 day, 12:36:43", "remaining_time": "0:17:26", "throughput": 458.21, "total_tokens": 60394224}
+{"current_steps": 39690, "total_steps": 40000, "loss": 0.3782, "lr": 7.457426147663982e-09, "epoch": 1.2704052237372767, "percentage": 99.22, "elapsed_time": "1 day, 12:36:45", "remaining_time": "0:17:09", "throughput": 458.26, "total_tokens": 60401792}
+{"current_steps": 39695, "total_steps": 40000, "loss": 0.5273, "lr": 7.219576535871797e-09, "epoch": 1.2705652647077652, "percentage": 99.24, "elapsed_time": "1 day, 12:36:48", "remaining_time": "0:16:52", "throughput": 458.31, "total_tokens": 60409024}
+{"current_steps": 39700, "total_steps": 40000, "loss": 0.511, "lr": 6.985581124896445e-09, "epoch": 1.2707253056782537, "percentage": 99.25, "elapsed_time": "1 day, 12:36:50", "remaining_time": "0:16:36", "throughput": 458.36, "total_tokens": 60416000}
+{"current_steps": 39705, "total_steps": 40000, "loss": 0.4961, "lr": 6.755439950828501e-09, "epoch": 1.270885346648742, "percentage": 99.26, "elapsed_time": "1 day, 12:36:52", "remaining_time": "0:16:19", "throughput": 458.41, "total_tokens": 60423808}
+{"current_steps": 39710, "total_steps": 40000, "loss": 0.5448, "lr": 6.5291530491562444e-09, "epoch": 1.2710453876192305, "percentage": 99.28, "elapsed_time": "1 day, 12:36:55", "remaining_time": "0:16:02", "throughput": 458.46, "total_tokens": 60431520}
+{"current_steps": 39715, "total_steps": 40000, "loss": 0.6854, "lr": 6.3067204547739845e-09, "epoch": 1.271205428589719, "percentage": 99.29, "elapsed_time": "1 day, 12:36:57", "remaining_time": "0:15:45", "throughput": 458.5, "total_tokens": 60438576}
+{"current_steps": 39720, "total_steps": 40000, "loss": 0.5586, "lr": 6.088142201987612e-09, "epoch": 1.2713654695602075, "percentage": 99.3, "elapsed_time": "1 day, 12:37:00", "remaining_time": "0:15:29", "throughput": 458.55, "total_tokens": 60446400}
+{"current_steps": 39725, "total_steps": 40000, "loss": 0.534, "lr": 5.873418324503499e-09, "epoch": 1.271525510530696, "percentage": 99.31, "elapsed_time": "1 day, 12:37:02", "remaining_time": "0:15:12", "throughput": 458.6, "total_tokens": 60453472}
+{"current_steps": 39730, "total_steps": 40000, "loss": 0.5117, "lr": 5.6625488554340465e-09, "epoch": 1.2716855515011842, "percentage": 99.33, "elapsed_time": "1 day, 12:37:04", "remaining_time": "0:14:55", "throughput": 458.64, "total_tokens": 60460576}
+{"current_steps": 39735, "total_steps": 40000, "loss": 0.3857, "lr": 5.455533827297688e-09, "epoch": 1.2718455924716727, "percentage": 99.34, "elapsed_time": "1 day, 12:37:07", "remaining_time": "0:14:39", "throughput": 458.69, "total_tokens": 60468416}
+{"current_steps": 39740, "total_steps": 40000, "loss": 0.5418, "lr": 5.252373272018885e-09, "epoch": 1.2720056334421612, "percentage": 99.35, "elapsed_time": "1 day, 12:37:09", "remaining_time": "0:14:22", "throughput": 458.74, "total_tokens": 60476128}
+{"current_steps": 39745, "total_steps": 40000, "loss": 0.5357, "lr": 5.053067220925356e-09, "epoch": 1.2721656744126497, "percentage": 99.36, "elapsed_time": "1 day, 12:37:12", "remaining_time": "0:14:05", "throughput": 458.79, "total_tokens": 60483776}
+{"current_steps": 39750, "total_steps": 40000, "loss": 0.557, "lr": 4.857615704759177e-09, "epoch": 1.272325715383138, "percentage": 99.38, "elapsed_time": "1 day, 12:37:14", "remaining_time": "0:13:49", "throughput": 458.84, "total_tokens": 60491248}
+{"current_steps": 39755, "total_steps": 40000, "loss": 0.3927, "lr": 4.666018753654577e-09, "epoch": 1.2724857563536265, "percentage": 99.39, "elapsed_time": "1 day, 12:37:17", "remaining_time": "0:13:32", "throughput": 458.89, "total_tokens": 60498992}
+{"current_steps": 39760, "total_steps": 40000, "loss": 0.4745, "lr": 4.478276397162917e-09, "epoch": 1.272645797324115, "percentage": 99.4, "elapsed_time": "1 day, 12:37:19", "remaining_time": "0:13:15", "throughput": 458.94, "total_tokens": 60506496}
+{"current_steps": 39765, "total_steps": 40000, "loss": 0.5125, "lr": 4.294388664233262e-09, "epoch": 1.2728058382946035, "percentage": 99.41, "elapsed_time": "1 day, 12:37:22", "remaining_time": "0:12:59", "throughput": 458.99, "total_tokens": 60513856}
+{"current_steps": 39770, "total_steps": 40000, "loss": 0.5183, "lr": 4.114355583223484e-09, "epoch": 1.272965879265092, "percentage": 99.42, "elapsed_time": "1 day, 12:37:24", "remaining_time": "0:12:42", "throughput": 459.03, "total_tokens": 60520832}
+{"current_steps": 39775, "total_steps": 40000, "loss": 0.4796, "lr": 3.9381771818974845e-09, "epoch": 1.2731259202355802, "percentage": 99.44, "elapsed_time": "1 day, 12:37:26", "remaining_time": "0:12:25", "throughput": 459.08, "total_tokens": 60528064}
+{"current_steps": 39780, "total_steps": 40000, "loss": 0.4709, "lr": 3.765853487427973e-09, "epoch": 1.2732859612060687, "percentage": 99.45, "elapsed_time": "1 day, 12:37:29", "remaining_time": "0:12:09", "throughput": 459.13, "total_tokens": 60535920}
+{"current_steps": 39785, "total_steps": 40000, "loss": 0.4565, "lr": 3.5973845263825857e-09, "epoch": 1.2734460021765572, "percentage": 99.46, "elapsed_time": "1 day, 12:37:31", "remaining_time": "0:11:52", "throughput": 459.18, "total_tokens": 60543776}
+{"current_steps": 39790, "total_steps": 40000, "loss": 0.4741, "lr": 3.4327703247488684e-09, "epoch": 1.2736060431470455, "percentage": 99.48, "elapsed_time": "1 day, 12:37:34", "remaining_time": "0:11:35", "throughput": 459.23, "total_tokens": 60551728}
+{"current_steps": 39795, "total_steps": 40000, "loss": 0.4521, "lr": 3.2720109079037443e-09, "epoch": 1.273766084117534, "percentage": 99.49, "elapsed_time": "1 day, 12:37:36", "remaining_time": "0:11:19", "throughput": 459.28, "total_tokens": 60559376}
+{"current_steps": 39800, "total_steps": 40000, "loss": 0.5532, "lr": 3.1151063006468193e-09, "epoch": 1.2739261250880225, "percentage": 99.5, "elapsed_time": "1 day, 12:37:39", "remaining_time": "0:11:02", "throughput": 459.33, "total_tokens": 60567296}
+{"current_steps": 39800, "total_steps": 40000, "eval_loss": 0.4941261410713196, "epoch": 1.2739261250880225, "percentage": 99.5, "elapsed_time": "1 day, 12:46:33", "remaining_time": "0:11:05", "throughput": 457.48, "total_tokens": 60567296}