jevkt-checkpoints / e15 /F2-flat /train_log.jsonl
Unggi's picture
backup e15/F2-flat (Pod B)
c23a7d2 verified
Raw History Blame Contribute Delete
8.67 kB
{"time": "00:48:41", "event": "start", "model_dir": "/workspace/jev_kt/models/hf/JEV-9B", "init": "released", "device": "cuda", "load_s": 16.315336227416992, "trainable_lora": 40108032, "head": 98328}
{"time": "00:48:41", "event": "run", "data": "data/e15/F2-flat", "out": "models/e15/F2-flat"}
{"time": "00:52:10", "event": "step", "step": 10, "rows_seen": 320, "loss": 0.5877577364444733, "lr": 7.333333333333333e-05, "tok_per_s": 855.269736009609}
{"time": "00:55:48", "event": "step", "step": 20, "rows_seen": 640, "loss": 0.5159218572080135, "lr": 9.997508952966197e-05, "tok_per_s": 860.6195371591518}
{"time": "00:59:26", "event": "step", "step": 30, "rows_seen": 960, "loss": 0.5548015087842941, "lr": 9.977596250541417e-05, "tok_per_s": 860.0880892463634}
{"time": "01:02:54", "event": "step", "step": 40, "rows_seen": 1280, "loss": 0.6564795449376106, "lr": 9.937854366485982e-05, "tok_per_s": 859.3455216368919}
{"time": "01:06:13", "event": "step", "step": 50, "rows_seen": 1600, "loss": 0.6747086718678474, "lr": 9.8784499920729e-05, "tok_per_s": 856.4053870753925}
{"time": "01:09:44", "event": "step", "step": 60, "rows_seen": 1920, "loss": 0.6810289993882179, "lr": 9.79963228989294e-05, "tok_per_s": 856.2979342247033}
{"time": "01:13:08", "event": "step", "step": 70, "rows_seen": 2240, "loss": 0.5894858688116074, "lr": 9.701731848780134e-05, "tok_per_s": 857.015999149776}
{"time": "01:16:36", "event": "step", "step": 80, "rows_seen": 2560, "loss": 0.5952880403201561, "lr": 9.585159297207315e-05, "tok_per_s": 857.0084571162397}
{"time": "01:20:10", "event": "step", "step": 90, "rows_seen": 2880, "loss": 0.5643837284296751, "lr": 9.450403580967612e-05, "tok_per_s": 857.0173718708572}
{"time": "01:23:28", "event": "step", "step": 100, "rows_seen": 3200, "loss": 0.6277467955369502, "lr": 9.298029912365855e-05, "tok_per_s": 856.8428464814812}
{"time": "01:26:56", "event": "step", "step": 110, "rows_seen": 3520, "loss": 0.5750151183456182, "lr": 9.128677399521752e-05, "tok_per_s": 857.3361248705531}
{"time": "01:30:32", "event": "step", "step": 120, "rows_seen": 3840, "loss": 0.6017883345484734, "lr": 8.943056365728335e-05, "tok_per_s": 857.6636177481919}
{"time": "01:34:02", "event": "step", "step": 130, "rows_seen": 4160, "loss": 0.5900653211865574, "lr": 8.741945370109202e-05, "tok_per_s": 857.7246233320602}
{"time": "01:37:25", "event": "step", "step": 140, "rows_seen": 4480, "loss": 0.5779878199100494, "lr": 8.526187942070997e-05, "tok_per_s": 857.9303305246107}
{"time": "01:40:43", "event": "step", "step": 150, "rows_seen": 4800, "loss": 0.617425411939621, "lr": 8.296689043247915e-05, "tok_per_s": 857.4952305884332}
{"time": "01:44:10", "event": "step", "step": 160, "rows_seen": 5120, "loss": 0.574973352253437, "lr": 8.054411271778132e-05, "tok_per_s": 857.8773445114482}
{"time": "01:47:36", "event": "step", "step": 170, "rows_seen": 5440, "loss": 0.5947388916974887, "lr": 7.800370824832674e-05, "tok_per_s": 858.5796525534176}
{"time": "01:51:04", "event": "step", "step": 180, "rows_seen": 5760, "loss": 0.6738452576100826, "lr": 7.535633236331324e-05, "tok_per_s": 858.7211851835583}
{"time": "01:54:38", "event": "step", "step": 190, "rows_seen": 6080, "loss": 0.5923438420286402, "lr": 7.26130890772308e-05, "tok_per_s": 858.5756695383079}
{"time": "01:58:04", "event": "step", "step": 200, "rows_seen": 6400, "loss": 0.6847539413720369, "lr": 6.978548450576568e-05, "tok_per_s": 858.6234518535314}
{"time": "02:01:44", "event": "step", "step": 210, "rows_seen": 6720, "loss": 0.6117992252111435, "lr": 6.688537860515266e-05, "tok_per_s": 859.1653949750619}
{"time": "02:05:10", "event": "step", "step": 220, "rows_seen": 7040, "loss": 0.6177883632481098, "lr": 6.392493542739706e-05, "tok_per_s": 859.2334070048599}
{"time": "02:08:33", "event": "step", "step": 230, "rows_seen": 7360, "loss": 0.5483945179730654, "lr": 6.091657210001309e-05, "tok_per_s": 859.0674583665049}
{"time": "02:12:03", "event": "step", "step": 240, "rows_seen": 7680, "loss": 0.622557744383812, "lr": 5.7872906744276015e-05, "tok_per_s": 859.0403296018193}
{"time": "02:15:33", "event": "step", "step": 250, "rows_seen": 8000, "loss": 0.6177436839789152, "lr": 5.48067055504363e-05, "tok_per_s": 858.9538478092975}
{"time": "02:19:02", "event": "step", "step": 260, "rows_seen": 8320, "loss": 0.6167473718523979, "lr": 5.173082923188119e-05, "tok_per_s": 858.6538598740115}
{"time": "02:22:32", "event": "step", "step": 270, "rows_seen": 8640, "loss": 0.5980293247848749, "lr": 4.865817908283331e-05, "tok_per_s": 858.8084109697298}
{"time": "02:26:01", "event": "step", "step": 280, "rows_seen": 8960, "loss": 0.5527693126350641, "lr": 4.560164286583922e-05, "tok_per_s": 858.8686163976956}
{"time": "02:29:33", "event": "step", "step": 290, "rows_seen": 9280, "loss": 0.5814823284745216, "lr": 4.257404075601443e-05, "tok_per_s": 858.8384409865034}
{"time": "02:33:01", "event": "step", "step": 300, "rows_seen": 9600, "loss": 0.5551043748855591, "lr": 3.958807156877371e-05, "tok_per_s": 859.1150473597955}
{"time": "02:36:31", "event": "step", "step": 310, "rows_seen": 9920, "loss": 0.697040218859911, "lr": 3.665625949658603e-05, "tok_per_s": 859.2355658129776}
{"time": "02:39:57", "event": "step", "step": 320, "rows_seen": 10240, "loss": 0.6119494661688805, "lr": 3.379090157815887e-05, "tok_per_s": 859.1054135607782}
{"time": "02:43:25", "event": "step", "step": 330, "rows_seen": 10560, "loss": 0.5685181058943272, "lr": 3.10040161203844e-05, "tok_per_s": 859.3126049036471}
{"time": "02:46:56", "event": "step", "step": 340, "rows_seen": 10880, "loss": 0.515652634203434, "lr": 2.8307292289383776e-05, "tok_per_s": 859.6799112780791}
{"time": "02:50:30", "event": "step", "step": 350, "rows_seen": 11200, "loss": 0.6127974279224873, "lr": 2.571204108208194e-05, "tok_per_s": 859.5528392833678}
{"time": "02:53:51", "event": "step", "step": 360, "rows_seen": 11520, "loss": 0.5996843129396439, "lr": 2.322914788395595e-05, "tok_per_s": 859.2857752821442}
{"time": "02:57:14", "event": "step", "step": 370, "rows_seen": 11840, "loss": 0.6045315861701965, "lr": 2.0869026811945157e-05, "tok_per_s": 859.3916305526038}
{"time": "03:00:48", "event": "step", "step": 380, "rows_seen": 12160, "loss": 0.6152966935187578, "lr": 1.8641577034025627e-05, "tok_per_s": 859.5163451459412}
{"time": "03:04:23", "event": "step", "step": 390, "rows_seen": 12480, "loss": 0.5862786285579205, "lr": 1.6556141248659472e-05, "tok_per_s": 859.431447094405}
{"time": "03:07:49", "event": "step", "step": 400, "rows_seen": 12800, "loss": 0.5740232467651367, "lr": 1.4621466498270773e-05, "tok_per_s": 859.4251833703263}
{"time": "03:11:20", "event": "step", "step": 410, "rows_seen": 13120, "loss": 0.5301701948046684, "lr": 1.284566748110963e-05, "tok_per_s": 859.393217107875}
{"time": "03:14:50", "event": "step", "step": 420, "rows_seen": 13440, "loss": 0.5579833164811134, "lr": 1.1236192515387821e-05, "tok_per_s": 859.4076421950874}
{"time": "03:18:15", "event": "step", "step": 430, "rows_seen": 13760, "loss": 0.6065985504537821, "lr": 9.799792298443728e-06, "tok_per_s": 859.3246262922454}
{"time": "03:21:43", "event": "step", "step": 440, "rows_seen": 14080, "loss": 0.6180005762726068, "lr": 8.542491591971554e-06, "tok_per_s": 859.356448017933}
{"time": "03:25:21", "event": "step", "step": 450, "rows_seen": 14400, "loss": 0.6323784813284874, "lr": 7.469563952077155e-06, "tok_per_s": 859.6901316662784}
{"time": "03:28:43", "event": "step", "step": 460, "rows_seen": 14720, "loss": 0.5437658387236297, "lr": 6.585509610150972e-06, "tok_per_s": 859.6373317863864}
{"time": "03:32:00", "event": "step", "step": 470, "rows_seen": 15040, "loss": 0.6128616854548454, "lr": 5.894036597333646e-06, "tok_per_s": 859.6926623129617}
{"time": "03:35:24", "event": "step", "step": 480, "rows_seen": 15360, "loss": 0.5803278815001249, "lr": 5.398045191744809e-06, "tok_per_s": 859.8447485778026}
{"time": "03:38:42", "event": "step", "step": 490, "rows_seen": 15680, "loss": 0.6114999353885651, "lr": 5.099615753708697e-06, "tok_per_s": 859.8744933018467}
{"time": "03:42:08", "event": "step", "step": 500, "rows_seen": 16000, "loss": 0.6015214323997498, "lr": 5e-06, "tok_per_s": 859.8938391996276}
{"time": "03:48:04", "event": "ckpt", "tag": "ckpt-16000", "rows_seen": 16000, "step": 500, "tokens": 8949001, "tok_per_s": 831.5855086824548, "val": {"n": 2000, "dropped_long": 0, "noul_ce": 0.5749193429946899, "kt_auc": 0.7322716618157148, "kt_acc": 0.683, "kt_f1": 0.7794015309672929, "kt_ece": 0.028756603643298143, "kt_nll": 0.5789322479655966, "kt_oracle_auc": 0.8438755004041573}, "dropped_long": 0, "peak_gib": 18.237350940704346}