| { |
| "config": { |
| "model": "LiquidAI/LFM2.5-Encoder-350M", |
| "dataset": "HuggingFaceFW/fineweb-edu", |
| "dataset_config": "sample-10BT", |
| "split": "train", |
| "text_column": "text", |
| "output_dir": "/tmp/minima", |
| "output_repo": null, |
| "sequence_length": 256, |
| "batch_size": 4, |
| "gradient_accumulation": 4, |
| "steps": 2000, |
| "learning_rate": 5e-05, |
| "warmup_steps": 100, |
| "group_size": 32, |
| "recovery_rank": 128, |
| "teacher_topk": 32, |
| "hidden_loss_weight": 2.0, |
| "distill_loss_weight": 1.0, |
| "mlm_loss_weight": 1.0, |
| "temperature": 2.0, |
| "seed": 42, |
| "log_every": 50, |
| "train_ternary_weights": false |
| }, |
| "duration_seconds": 667.6906430721283, |
| "trainable_parameters": 56492032, |
| "history": [ |
| { |
| "step": 1, |
| "lr": 5e-07, |
| "elapsed_s": 7.1458351612091064, |
| "loss": 7.548747301101685, |
| "hidden": 0.42470795661211014, |
| "mlm": 2.650780439376831, |
| "distill": 4.048550963401794, |
| "teacher_acc": 0.7796393632888794 |
| }, |
| { |
| "step": 51, |
| "lr": 2.5500000000000003e-05, |
| "elapsed_s": 25.010111331939697, |
| "loss": 4.090662062168121, |
| "hidden": 0.30477869510650635, |
| "mlm": 1.5020031332969666, |
| "distill": 1.9791015088558197, |
| "teacher_acc": 0.7966673225164413 |
| }, |
| { |
| "step": 101, |
| "lr": 5e-05, |
| "elapsed_s": 42.059675216674805, |
| "loss": 3.8598657846450806, |
| "hidden": 0.2699665352702141, |
| "mlm": 1.7310407161712646, |
| "distill": 1.5888920724391937, |
| "teacher_acc": 0.7436814606189728 |
| }, |
| { |
| "step": 151, |
| "lr": 4.991461232516675e-05, |
| "elapsed_s": 58.84036660194397, |
| "loss": 2.8976023197174072, |
| "hidden": 0.2286553680896759, |
| "mlm": 1.2483952790498734, |
| "distill": 1.191896229982376, |
| "teacher_acc": 0.8049547970294952 |
| }, |
| { |
| "step": 201, |
| "lr": 4.965903258506806e-05, |
| "elapsed_s": 75.38389277458191, |
| "loss": 2.984165906906128, |
| "hidden": 0.22897232323884964, |
| "mlm": 1.3140619993209839, |
| "distill": 1.2121593058109283, |
| "teacher_acc": 0.8042650818824768 |
| }, |
| { |
| "step": 251, |
| "lr": 4.923500664848326e-05, |
| "elapsed_s": 91.81105828285217, |
| "loss": 2.715985119342804, |
| "hidden": 0.19838016480207443, |
| "mlm": 1.3606167435646057, |
| "distill": 0.9586080610752106, |
| "teacher_acc": 0.7605845779180527 |
| }, |
| { |
| "step": 301, |
| "lr": 4.864543104251587e-05, |
| "elapsed_s": 108.18112087249756, |
| "loss": 2.683966338634491, |
| "hidden": 0.18938461691141129, |
| "mlm": 1.3027366399765015, |
| "distill": 1.002460464835167, |
| "teacher_acc": 0.7672062069177628 |
| }, |
| { |
| "step": 351, |
| "lr": 4.789433316637644e-05, |
| "elapsed_s": 124.41784739494324, |
| "loss": 2.9636711478233337, |
| "hidden": 0.19395725429058075, |
| "mlm": 1.370901107788086, |
| "distill": 1.204855501651764, |
| "teacher_acc": 0.7925134599208832 |
| }, |
| { |
| "step": 401, |
| "lr": 4.698684378016222e-05, |
| "elapsed_s": 140.6487717628479, |
| "loss": 2.5739916563034058, |
| "hidden": 0.1828564964234829, |
| "mlm": 1.262264758348465, |
| "distill": 0.9460138827562332, |
| "teacher_acc": 0.7860687077045441 |
| }, |
| { |
| "step": 451, |
| "lr": 4.592916195656322e-05, |
| "elapsed_s": 156.78596544265747, |
| "loss": 2.4724334478378296, |
| "hidden": 0.18117711693048477, |
| "mlm": 1.1873478889465332, |
| "distill": 0.9227313250303268, |
| "teacher_acc": 0.8220411986112595 |
| }, |
| { |
| "step": 501, |
| "lr": 4.4728512734909844e-05, |
| "elapsed_s": 173.03300976753235, |
| "loss": 2.516365110874176, |
| "hidden": 0.18044547364115715, |
| "mlm": 1.1758874356746674, |
| "distill": 0.9795867055654526, |
| "teacher_acc": 0.7911238372325897 |
| }, |
| { |
| "step": 551, |
| "lr": 4.3393097766828293e-05, |
| "elapsed_s": 189.45385837554932, |
| "loss": 2.470432221889496, |
| "hidden": 0.17229371145367622, |
| "mlm": 1.2614093571901321, |
| "distill": 0.864435464143753, |
| "teacher_acc": 0.7440680414438248 |
| }, |
| { |
| "step": 601, |
| "lr": 4.193203929064353e-05, |
| "elapsed_s": 205.8019621372223, |
| "loss": 2.225749373435974, |
| "hidden": 0.17066730186343193, |
| "mlm": 1.0380512028932571, |
| "distill": 0.8463635146617889, |
| "teacher_acc": 0.7945135682821274 |
| }, |
| { |
| "step": 651, |
| "lr": 4.03553178172417e-05, |
| "elapsed_s": 222.18287634849548, |
| "loss": 2.412706196308136, |
| "hidden": 0.1683909185230732, |
| "mlm": 1.1580713838338852, |
| "distill": 0.9178528636693954, |
| "teacher_acc": 0.7857213616371155 |
| }, |
| { |
| "step": 701, |
| "lr": 3.867370395306068e-05, |
| "elapsed_s": 238.73087334632874, |
| "loss": 2.531893312931061, |
| "hidden": 0.1756228394806385, |
| "mlm": 1.2564030885696411, |
| "distill": 0.9242444187402725, |
| "teacher_acc": 0.7829260528087616 |
| }, |
| { |
| "step": 751, |
| "lr": 3.689868482592684e-05, |
| "elapsed_s": 254.98722314834595, |
| "loss": 2.4058578610420227, |
| "hidden": 0.1706235148012638, |
| "mlm": 1.092416599392891, |
| "distill": 0.9721942096948624, |
| "teacher_acc": 0.807672306895256 |
| }, |
| { |
| "step": 801, |
| "lr": 3.504238561632424e-05, |
| "elapsed_s": 271.2415738105774, |
| "loss": 2.4489212036132812, |
| "hidden": 0.16438886150717735, |
| "mlm": 1.249473124742508, |
| "distill": 0.8706703633069992, |
| "teacher_acc": 0.7654353976249695 |
| }, |
| { |
| "step": 851, |
| "lr": 3.311748673011709e-05, |
| "elapsed_s": 287.5516035556793, |
| "loss": 2.139587104320526, |
| "hidden": 0.16303318738937378, |
| "mlm": 1.0034526735544205, |
| "distill": 0.8100681155920029, |
| "teacher_acc": 0.8189881294965744 |
| }, |
| { |
| "step": 901, |
| "lr": 3.1137137178519985e-05, |
| "elapsed_s": 303.8067092895508, |
| "loss": 2.2904033064842224, |
| "hidden": 0.1645217277109623, |
| "mlm": 1.1553480178117752, |
| "distill": 0.8060119301080704, |
| "teacher_acc": 0.7967804372310638 |
| }, |
| { |
| "step": 951, |
| "lr": 2.9114864757018352e-05, |
| "elapsed_s": 320.1048364639282, |
| "loss": 2.320096254348755, |
| "hidden": 0.1607908457517624, |
| "mlm": 1.1159868836402893, |
| "distill": 0.8825277388095856, |
| "teacher_acc": 0.7653505355119705 |
| }, |
| { |
| "step": 1001, |
| "lr": 2.7064483636808313e-05, |
| "elapsed_s": 336.35610246658325, |
| "loss": 2.4431122541427612, |
| "hidden": 0.1690278798341751, |
| "mlm": 1.2797610759735107, |
| "distill": 0.825295478105545, |
| "teacher_acc": 0.7673374265432358 |
| }, |
| { |
| "step": 1051, |
| "lr": 2.5e-05, |
| "elapsed_s": 352.62019753456116, |
| "loss": 2.4243820905685425, |
| "hidden": 0.15917402133345604, |
| "mlm": 1.2654687762260437, |
| "distill": 0.8405653089284897, |
| "teacher_acc": 0.7821898609399796 |
| }, |
| { |
| "step": 1101, |
| "lr": 2.2935516363191693e-05, |
| "elapsed_s": 368.93653750419617, |
| "loss": 2.39467990398407, |
| "hidden": 0.1649530865252018, |
| "mlm": 1.2698808908462524, |
| "distill": 0.7948928475379944, |
| "teacher_acc": 0.7567200213670731 |
| }, |
| { |
| "step": 1151, |
| "lr": 2.088513524298165e-05, |
| "elapsed_s": 385.3177971839905, |
| "loss": 2.786853849887848, |
| "hidden": 0.17868195474147797, |
| "mlm": 1.416623666882515, |
| "distill": 1.0128663033246994, |
| "teacher_acc": 0.747254803776741 |
| }, |
| { |
| "step": 1201, |
| "lr": 1.8862862821480025e-05, |
| "elapsed_s": 401.6415090560913, |
| "loss": 2.155725210905075, |
| "hidden": 0.15804878994822502, |
| "mlm": 0.908401682972908, |
| "distill": 0.9312258809804916, |
| "teacher_acc": 0.8400556594133377 |
| }, |
| { |
| "step": 1251, |
| "lr": 1.6882513269882917e-05, |
| "elapsed_s": 417.91031408309937, |
| "loss": 2.757974684238434, |
| "hidden": 0.16840778663754463, |
| "mlm": 1.393098160624504, |
| "distill": 1.0280609875917435, |
| "teacher_acc": 0.755160465836525 |
| }, |
| { |
| "step": 1301, |
| "lr": 1.495761438367577e-05, |
| "elapsed_s": 434.33623337745667, |
| "loss": 2.444422423839569, |
| "hidden": 0.15633703023195267, |
| "mlm": 1.2066814005374908, |
| "distill": 0.9250669926404953, |
| "teacher_acc": 0.7845637202262878 |
| }, |
| { |
| "step": 1351, |
| "lr": 1.3101315174073162e-05, |
| "elapsed_s": 450.75909185409546, |
| "loss": 2.466778337955475, |
| "hidden": 0.16462653502821922, |
| "mlm": 1.2565617859363556, |
| "distill": 0.880963459610939, |
| "teacher_acc": 0.762797936797142 |
| }, |
| { |
| "step": 1401, |
| "lr": 1.1326296046939333e-05, |
| "elapsed_s": 467.06108593940735, |
| "loss": 2.127809077501297, |
| "hidden": 0.1562965139746666, |
| "mlm": 1.076869934797287, |
| "distill": 0.7383460700511932, |
| "teacher_acc": 0.79219089448452 |
| }, |
| { |
| "step": 1451, |
| "lr": 9.644682182758306e-06, |
| "elapsed_s": 483.31524443626404, |
| "loss": 2.3861597776412964, |
| "hidden": 0.15558123216032982, |
| "mlm": 1.28284552693367, |
| "distill": 0.7921517193317413, |
| "teacher_acc": 0.7626918703317642 |
| }, |
| { |
| "step": 1501, |
| "lr": 8.067960709356478e-06, |
| "elapsed_s": 499.5990273952484, |
| "loss": 2.2876625657081604, |
| "hidden": 0.15779289230704308, |
| "mlm": 1.210097774863243, |
| "distill": 0.7619789987802505, |
| "teacher_acc": 0.7711311429738998 |
| }, |
| { |
| "step": 1551, |
| "lr": 6.606902233171711e-06, |
| "elapsed_s": 515.7814161777496, |
| "loss": 1.987428605556488, |
| "hidden": 0.1505429930984974, |
| "mlm": 0.9722132682800293, |
| "distill": 0.7141293436288834, |
| "teacher_acc": 0.81315678358078 |
| }, |
| { |
| "step": 1601, |
| "lr": 5.271487265090163e-06, |
| "elapsed_s": 531.9731659889221, |
| "loss": 2.178088665008545, |
| "hidden": 0.1585002951323986, |
| "mlm": 1.1130180209875107, |
| "distill": 0.7480701059103012, |
| "teacher_acc": 0.7750980257987976 |
| }, |
| { |
| "step": 1651, |
| "lr": 4.070838043436786e-06, |
| "elapsed_s": 549.6739189624786, |
| "loss": 2.2565123438835144, |
| "hidden": 0.15409696847200394, |
| "mlm": 1.1320631355047226, |
| "distill": 0.8162552565336227, |
| "teacher_acc": 0.7810630649328232 |
| }, |
| { |
| "step": 1701, |
| "lr": 3.013156219837776e-06, |
| "elapsed_s": 569.2521679401398, |
| "loss": 2.5504953861236572, |
| "hidden": 0.1535499133169651, |
| "mlm": 1.3510537147521973, |
| "distill": 0.8923418670892715, |
| "teacher_acc": 0.7388543039560318 |
| }, |
| { |
| "step": 1751, |
| "lr": 2.1056668336235622e-06, |
| "elapsed_s": 585.5749201774597, |
| "loss": 2.126615345478058, |
| "hidden": 0.15592899918556213, |
| "mlm": 1.0641395151615143, |
| "distill": 0.7506178617477417, |
| "teacher_acc": 0.8043123483657837 |
| }, |
| { |
| "step": 1801, |
| "lr": 1.3545689574841342e-06, |
| "elapsed_s": 602.0534913539886, |
| "loss": 2.315759599208832, |
| "hidden": 0.15497129783034325, |
| "mlm": 1.1510487794876099, |
| "distill": 0.8547682464122772, |
| "teacher_acc": 0.7958026677370071 |
| }, |
| { |
| "step": 1851, |
| "lr": 7.649933515167407e-07, |
| "elapsed_s": 618.2598822116852, |
| "loss": 2.6134771704673767, |
| "hidden": 0.16912326216697693, |
| "mlm": 1.3344760239124298, |
| "distill": 0.9407546371221542, |
| "teacher_acc": 0.7435210645198822 |
| }, |
| { |
| "step": 1901, |
| "lr": 3.4096741493194197e-07, |
| "elapsed_s": 634.6096048355103, |
| "loss": 2.525772988796234, |
| "hidden": 0.15609804540872574, |
| "mlm": 1.3272225856781006, |
| "distill": 0.8863543570041656, |
| "teacher_acc": 0.7534686625003815 |
| }, |
| { |
| "step": 1951, |
| "lr": 8.538767483325383e-08, |
| "elapsed_s": 650.8219337463379, |
| "loss": 2.462930738925934, |
| "hidden": 0.15661108866333961, |
| "mlm": 1.2163092494010925, |
| "distill": 0.9333993047475815, |
| "teacher_acc": 0.7857980132102966 |
| }, |
| { |
| "step": 2000, |
| "lr": 3.417452268950072e-11, |
| "elapsed_s": 666.7361209392548, |
| "loss": 2.3209976255893707, |
| "hidden": 0.15706806257367134, |
| "mlm": 1.1457223743200302, |
| "distill": 0.8611391484737396, |
| "teacher_acc": 0.7896232903003693 |
| } |
| ] |
| } |
|
|
