training_report.json

{
  "config": {
    "model": "LiquidAI/LFM2.5-Encoder-350M",
    "dataset": "HuggingFaceFW/fineweb-edu",
    "dataset_config": "sample-10BT",
    "split": "train",
    "text_column": "text",
    "output_dir": "/tmp/minima",
    "output_repo": null,
    "sequence_length": 256,
    "batch_size": 4,
    "gradient_accumulation": 4,
    "steps": 2000,
    "learning_rate": 5e-05,
    "warmup_steps": 100,
    "group_size": 32,
    "recovery_rank": 128,
    "teacher_topk": 32,
    "hidden_loss_weight": 2.0,
    "distill_loss_weight": 1.0,
    "mlm_loss_weight": 1.0,
    "temperature": 2.0,
    "seed": 42,
    "log_every": 50,
    "train_ternary_weights": false
  },
  "duration_seconds": 667.6906430721283,
  "trainable_parameters": 56492032,
  "history": [
    {
      "step": 1,
      "lr": 5e-07,
      "elapsed_s": 7.1458351612091064,
      "loss": 7.548747301101685,
      "hidden": 0.42470795661211014,
      "mlm": 2.650780439376831,
      "distill": 4.048550963401794,
      "teacher_acc": 0.7796393632888794
    },
    {
      "step": 51,
      "lr": 2.5500000000000003e-05,
      "elapsed_s": 25.010111331939697,
      "loss": 4.090662062168121,
      "hidden": 0.30477869510650635,
      "mlm": 1.5020031332969666,
      "distill": 1.9791015088558197,
      "teacher_acc": 0.7966673225164413
    },
    {
      "step": 101,
      "lr": 5e-05,
      "elapsed_s": 42.059675216674805,
      "loss": 3.8598657846450806,
      "hidden": 0.2699665352702141,
      "mlm": 1.7310407161712646,
      "distill": 1.5888920724391937,
      "teacher_acc": 0.7436814606189728
    },
    {
      "step": 151,
      "lr": 4.991461232516675e-05,
      "elapsed_s": 58.84036660194397,
      "loss": 2.8976023197174072,
      "hidden": 0.2286553680896759,
      "mlm": 1.2483952790498734,
      "distill": 1.191896229982376,
      "teacher_acc": 0.8049547970294952
    },
    {
      "step": 201,
      "lr": 4.965903258506806e-05,
      "elapsed_s": 75.38389277458191,
      "loss": 2.984165906906128,
      "hidden": 0.22897232323884964,
      "mlm": 1.3140619993209839,
      "distill": 1.2121593058109283,
      "teacher_acc": 0.8042650818824768
    },
    {
      "step": 251,
      "lr": 4.923500664848326e-05,
      "elapsed_s": 91.81105828285217,
      "loss": 2.715985119342804,
      "hidden": 0.19838016480207443,
      "mlm": 1.3606167435646057,
      "distill": 0.9586080610752106,
      "teacher_acc": 0.7605845779180527
    },
    {
      "step": 301,
      "lr": 4.864543104251587e-05,
      "elapsed_s": 108.18112087249756,
      "loss": 2.683966338634491,
      "hidden": 0.18938461691141129,
      "mlm": 1.3027366399765015,
      "distill": 1.002460464835167,
      "teacher_acc": 0.7672062069177628
    },
    {
      "step": 351,
      "lr": 4.789433316637644e-05,
      "elapsed_s": 124.41784739494324,
      "loss": 2.9636711478233337,
      "hidden": 0.19395725429058075,
      "mlm": 1.370901107788086,
      "distill": 1.204855501651764,
      "teacher_acc": 0.7925134599208832
    },
    {
      "step": 401,
      "lr": 4.698684378016222e-05,
      "elapsed_s": 140.6487717628479,
      "loss": 2.5739916563034058,
      "hidden": 0.1828564964234829,
      "mlm": 1.262264758348465,
      "distill": 0.9460138827562332,
      "teacher_acc": 0.7860687077045441
    },
    {
      "step": 451,
      "lr": 4.592916195656322e-05,
      "elapsed_s": 156.78596544265747,
      "loss": 2.4724334478378296,
      "hidden": 0.18117711693048477,
      "mlm": 1.1873478889465332,
      "distill": 0.9227313250303268,
      "teacher_acc": 0.8220411986112595
    },
    {
      "step": 501,
      "lr": 4.4728512734909844e-05,
      "elapsed_s": 173.03300976753235,
      "loss": 2.516365110874176,
      "hidden": 0.18044547364115715,
      "mlm": 1.1758874356746674,
      "distill": 0.9795867055654526,
      "teacher_acc": 0.7911238372325897
    },
    {
      "step": 551,
      "lr": 4.3393097766828293e-05,
      "elapsed_s": 189.45385837554932,
      "loss": 2.470432221889496,
      "hidden": 0.17229371145367622,
      "mlm": 1.2614093571901321,
      "distill": 0.864435464143753,
      "teacher_acc": 0.7440680414438248
    },
    {
      "step": 601,
      "lr": 4.193203929064353e-05,
      "elapsed_s": 205.8019621372223,
      "loss": 2.225749373435974,
      "hidden": 0.17066730186343193,
      "mlm": 1.0380512028932571,
      "distill": 0.8463635146617889,
      "teacher_acc": 0.7945135682821274
    },
    {
      "step": 651,
      "lr": 4.03553178172417e-05,
      "elapsed_s": 222.18287634849548,
      "loss": 2.412706196308136,
      "hidden": 0.1683909185230732,
      "mlm": 1.1580713838338852,
      "distill": 0.9178528636693954,
      "teacher_acc": 0.7857213616371155
    },
    {
      "step": 701,
      "lr": 3.867370395306068e-05,
      "elapsed_s": 238.73087334632874,
      "loss": 2.531893312931061,
      "hidden": 0.1756228394806385,
      "mlm": 1.2564030885696411,
      "distill": 0.9242444187402725,
      "teacher_acc": 0.7829260528087616
    },
    {
      "step": 751,
      "lr": 3.689868482592684e-05,
      "elapsed_s": 254.98722314834595,
      "loss": 2.4058578610420227,
      "hidden": 0.1706235148012638,
      "mlm": 1.092416599392891,
      "distill": 0.9721942096948624,
      "teacher_acc": 0.807672306895256
    },
    {
      "step": 801,
      "lr": 3.504238561632424e-05,
      "elapsed_s": 271.2415738105774,
      "loss": 2.4489212036132812,
      "hidden": 0.16438886150717735,
      "mlm": 1.249473124742508,
      "distill": 0.8706703633069992,
      "teacher_acc": 0.7654353976249695
    },
    {
      "step": 851,
      "lr": 3.311748673011709e-05,
      "elapsed_s": 287.5516035556793,
      "loss": 2.139587104320526,
      "hidden": 0.16303318738937378,
      "mlm": 1.0034526735544205,
      "distill": 0.8100681155920029,
      "teacher_acc": 0.8189881294965744
    },
    {
      "step": 901,
      "lr": 3.1137137178519985e-05,
      "elapsed_s": 303.8067092895508,
      "loss": 2.2904033064842224,
      "hidden": 0.1645217277109623,
      "mlm": 1.1553480178117752,
      "distill": 0.8060119301080704,
      "teacher_acc": 0.7967804372310638
    },
    {
      "step": 951,
      "lr": 2.9114864757018352e-05,
      "elapsed_s": 320.1048364639282,
      "loss": 2.320096254348755,
      "hidden": 0.1607908457517624,
      "mlm": 1.1159868836402893,
      "distill": 0.8825277388095856,
      "teacher_acc": 0.7653505355119705
    },
    {
      "step": 1001,
      "lr": 2.7064483636808313e-05,
      "elapsed_s": 336.35610246658325,
      "loss": 2.4431122541427612,
      "hidden": 0.1690278798341751,
      "mlm": 1.2797610759735107,
      "distill": 0.825295478105545,
      "teacher_acc": 0.7673374265432358
    },
    {
      "step": 1051,
      "lr": 2.5e-05,
      "elapsed_s": 352.62019753456116,
      "loss": 2.4243820905685425,
      "hidden": 0.15917402133345604,
      "mlm": 1.2654687762260437,
      "distill": 0.8405653089284897,
      "teacher_acc": 0.7821898609399796
    },
    {
      "step": 1101,
      "lr": 2.2935516363191693e-05,
      "elapsed_s": 368.93653750419617,
      "loss": 2.39467990398407,
      "hidden": 0.1649530865252018,
      "mlm": 1.2698808908462524,
      "distill": 0.7948928475379944,
      "teacher_acc": 0.7567200213670731
    },
    {
      "step": 1151,
      "lr": 2.088513524298165e-05,
      "elapsed_s": 385.3177971839905,
      "loss": 2.786853849887848,
      "hidden": 0.17868195474147797,
      "mlm": 1.416623666882515,
      "distill": 1.0128663033246994,
      "teacher_acc": 0.747254803776741
    },
    {
      "step": 1201,
      "lr": 1.8862862821480025e-05,
      "elapsed_s": 401.6415090560913,
      "loss": 2.155725210905075,
      "hidden": 0.15804878994822502,
      "mlm": 0.908401682972908,
      "distill": 0.9312258809804916,
      "teacher_acc": 0.8400556594133377
    },
    {
      "step": 1251,
      "lr": 1.6882513269882917e-05,
      "elapsed_s": 417.91031408309937,
      "loss": 2.757974684238434,
      "hidden": 0.16840778663754463,
      "mlm": 1.393098160624504,
      "distill": 1.0280609875917435,
      "teacher_acc": 0.755160465836525
    },
    {
      "step": 1301,
      "lr": 1.495761438367577e-05,
      "elapsed_s": 434.33623337745667,
      "loss": 2.444422423839569,
      "hidden": 0.15633703023195267,
      "mlm": 1.2066814005374908,
      "distill": 0.9250669926404953,
      "teacher_acc": 0.7845637202262878
    },
    {
      "step": 1351,
      "lr": 1.3101315174073162e-05,
      "elapsed_s": 450.75909185409546,
      "loss": 2.466778337955475,
      "hidden": 0.16462653502821922,
      "mlm": 1.2565617859363556,
      "distill": 0.880963459610939,
      "teacher_acc": 0.762797936797142
    },
    {
      "step": 1401,
      "lr": 1.1326296046939333e-05,
      "elapsed_s": 467.06108593940735,
      "loss": 2.127809077501297,
      "hidden": 0.1562965139746666,
      "mlm": 1.076869934797287,
      "distill": 0.7383460700511932,
      "teacher_acc": 0.79219089448452
    },
    {
      "step": 1451,
      "lr": 9.644682182758306e-06,
      "elapsed_s": 483.31524443626404,
      "loss": 2.3861597776412964,
      "hidden": 0.15558123216032982,
      "mlm": 1.28284552693367,
      "distill": 0.7921517193317413,
      "teacher_acc": 0.7626918703317642
    },
    {
      "step": 1501,
      "lr": 8.067960709356478e-06,
      "elapsed_s": 499.5990273952484,
      "loss": 2.2876625657081604,
      "hidden": 0.15779289230704308,
      "mlm": 1.210097774863243,
      "distill": 0.7619789987802505,
      "teacher_acc": 0.7711311429738998
    },
    {
      "step": 1551,
      "lr": 6.606902233171711e-06,
      "elapsed_s": 515.7814161777496,
      "loss": 1.987428605556488,
      "hidden": 0.1505429930984974,
      "mlm": 0.9722132682800293,
      "distill": 0.7141293436288834,
      "teacher_acc": 0.81315678358078
    },
    {
      "step": 1601,
      "lr": 5.271487265090163e-06,
      "elapsed_s": 531.9731659889221,
      "loss": 2.178088665008545,
      "hidden": 0.1585002951323986,
      "mlm": 1.1130180209875107,
      "distill": 0.7480701059103012,
      "teacher_acc": 0.7750980257987976
    },
    {
      "step": 1651,
      "lr": 4.070838043436786e-06,
      "elapsed_s": 549.6739189624786,
      "loss": 2.2565123438835144,
      "hidden": 0.15409696847200394,
      "mlm": 1.1320631355047226,
      "distill": 0.8162552565336227,
      "teacher_acc": 0.7810630649328232
    },
    {
      "step": 1701,
      "lr": 3.013156219837776e-06,
      "elapsed_s": 569.2521679401398,
      "loss": 2.5504953861236572,
      "hidden": 0.1535499133169651,
      "mlm": 1.3510537147521973,
      "distill": 0.8923418670892715,
      "teacher_acc": 0.7388543039560318
    },
    {
      "step": 1751,
      "lr": 2.1056668336235622e-06,
      "elapsed_s": 585.5749201774597,
      "loss": 2.126615345478058,
      "hidden": 0.15592899918556213,
      "mlm": 1.0641395151615143,
      "distill": 0.7506178617477417,
      "teacher_acc": 0.8043123483657837
    },
    {
      "step": 1801,
      "lr": 1.3545689574841342e-06,
      "elapsed_s": 602.0534913539886,
      "loss": 2.315759599208832,
      "hidden": 0.15497129783034325,
      "mlm": 1.1510487794876099,
      "distill": 0.8547682464122772,
      "teacher_acc": 0.7958026677370071
    },
    {
      "step": 1851,
      "lr": 7.649933515167407e-07,
      "elapsed_s": 618.2598822116852,
      "loss": 2.6134771704673767,
      "hidden": 0.16912326216697693,
      "mlm": 1.3344760239124298,
      "distill": 0.9407546371221542,
      "teacher_acc": 0.7435210645198822
    },
    {
      "step": 1901,
      "lr": 3.4096741493194197e-07,
      "elapsed_s": 634.6096048355103,
      "loss": 2.525772988796234,
      "hidden": 0.15609804540872574,
      "mlm": 1.3272225856781006,
      "distill": 0.8863543570041656,
      "teacher_acc": 0.7534686625003815
    },
    {
      "step": 1951,
      "lr": 8.538767483325383e-08,
      "elapsed_s": 650.8219337463379,
      "loss": 2.462930738925934,
      "hidden": 0.15661108866333961,
      "mlm": 1.2163092494010925,
      "distill": 0.9333993047475815,
      "teacher_acc": 0.7857980132102966
    },
    {
      "step": 2000,
      "lr": 3.417452268950072e-11,
      "elapsed_s": 666.7361209392548,
      "loss": 2.3209976255893707,
      "hidden": 0.15706806257367134,
      "mlm": 1.1457223743200302,
      "distill": 0.8611391484737396,
      "teacher_acc": 0.7896232903003693
    }
  ]
}