{ "best_global_step": 7450, "best_metric": 1.0779017210006714, "best_model_checkpoint": "./CanonBert/checkpoint-7450", "epoch": 10.0, "eval_steps": 500, "global_step": 7450, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.001342957864696995, "grad_norm": 31.485458374023438, "learning_rate": 0.0, "loss": 32.58583068847656, "num_input_tokens_seen": 327680, "step": 1, "train_runtime": 8.0777, "train_tokens_per_second": 40566.209 }, { "epoch": 0.06714789323484975, "grad_norm": 29.44062614440918, "learning_rate": 1.0961968680089486e-05, "loss": 32.123365752551024, "num_input_tokens_seen": 16384000, "step": 50, "train_runtime": 382.1108, "train_tokens_per_second": 42877.616 }, { "epoch": 0.1342957864696995, "grad_norm": 30.11578369140625, "learning_rate": 2.2147651006711412e-05, "loss": 31.29671875, "num_input_tokens_seen": 32768000, "step": 100, "train_runtime": 764.7247, "train_tokens_per_second": 42849.407 }, { "epoch": 0.20144367970454927, "grad_norm": 31.309894561767578, "learning_rate": 3.3333333333333335e-05, "loss": 30.311025390625, "num_input_tokens_seen": 49152000, "step": 150, "train_runtime": 1146.9476, "train_tokens_per_second": 42854.617 }, { "epoch": 0.268591572939399, "grad_norm": 34.5892448425293, "learning_rate": 4.451901565995526e-05, "loss": 29.528154296875, "num_input_tokens_seen": 65536000, "step": 200, "train_runtime": 1528.6613, "train_tokens_per_second": 42871.499 }, { "epoch": 0.33573946617424877, "grad_norm": 32.915374755859375, "learning_rate": 5.570469798657718e-05, "loss": 28.81987060546875, "num_input_tokens_seen": 81920000, "step": 250, "train_runtime": 1910.4025, "train_tokens_per_second": 42881.016 }, { "epoch": 0.40288735940909853, "grad_norm": 33.54853057861328, "learning_rate": 6.689038031319911e-05, "loss": 28.16813232421875, "num_input_tokens_seen": 98304000, "step": 300, "train_runtime": 2292.7173, "train_tokens_per_second": 42876.633 }, { "epoch": 0.4700352526439483, "grad_norm": 32.1806526184082, "learning_rate": 7.807606263982104e-05, "loss": 27.5452734375, "num_input_tokens_seen": 114688000, "step": 350, "train_runtime": 2675.3376, "train_tokens_per_second": 42868.608 }, { "epoch": 0.537183145878798, "grad_norm": 31.719738006591797, "learning_rate": 8.926174496644296e-05, "loss": 27.00053955078125, "num_input_tokens_seen": 131072000, "step": 400, "train_runtime": 3057.9475, "train_tokens_per_second": 42862.738 }, { "epoch": 0.6043310391136478, "grad_norm": 32.894378662109375, "learning_rate": 9.997144081108098e-05, "loss": 26.5186083984375, "num_input_tokens_seen": 147456000, "step": 450, "train_runtime": 3440.035, "train_tokens_per_second": 42864.681 }, { "epoch": 0.6714789323484975, "grad_norm": 31.206104278564453, "learning_rate": 9.925746108810509e-05, "loss": 25.94174072265625, "num_input_tokens_seen": 163840000, "step": 500, "train_runtime": 3822.0237, "train_tokens_per_second": 42867.343 }, { "epoch": 0.6714789323484975, "eval_loss": 1.5885820388793945, "eval_masked_accuracy": 0.680313674457695, "eval_runtime": 18.1032, "eval_samples_per_second": 217.254, "eval_steps_per_second": 3.425, "num_input_tokens_seen": 163840000, "step": 500 }, { "epoch": 0.7386268255833474, "grad_norm": 28.808670043945312, "learning_rate": 9.854348136512923e-05, "loss": 25.40889404296875, "num_input_tokens_seen": 180224000, "step": 550, "train_runtime": 4224.9229, "train_tokens_per_second": 42657.347 }, { "epoch": 0.8057747188181971, "grad_norm": 27.676965713500977, "learning_rate": 9.782950164215337e-05, "loss": 25.1195703125, "num_input_tokens_seen": 196608000, "step": 600, "train_runtime": 4606.8011, "train_tokens_per_second": 42677.771 }, { "epoch": 0.8729226120530469, "grad_norm": 26.387683868408203, "learning_rate": 9.71155219191775e-05, "loss": 24.66832275390625, "num_input_tokens_seen": 212992000, "step": 650, "train_runtime": 4988.9391, "train_tokens_per_second": 42692.844 }, { "epoch": 0.9400705052878966, "grad_norm": 26.794294357299805, "learning_rate": 9.640154219620163e-05, "loss": 24.388662109375, "num_input_tokens_seen": 229376000, "step": 700, "train_runtime": 5371.3616, "train_tokens_per_second": 42703.511 }, { "epoch": 1.006714789323485, "grad_norm": 26.925813674926758, "learning_rate": 9.568756247322577e-05, "loss": 23.8973291015625, "num_input_tokens_seen": 245617664, "step": 750, "train_runtime": 5750.9923, "train_tokens_per_second": 42708.745 }, { "epoch": 1.0738626825583348, "grad_norm": 28.082378387451172, "learning_rate": 9.49735827502499e-05, "loss": 23.6507275390625, "num_input_tokens_seen": 262001664, "step": 800, "train_runtime": 6133.6157, "train_tokens_per_second": 42715.696 }, { "epoch": 1.1410105757931845, "grad_norm": 29.90462303161621, "learning_rate": 9.425960302727404e-05, "loss": 23.39791015625, "num_input_tokens_seen": 278385664, "step": 850, "train_runtime": 6516.0776, "train_tokens_per_second": 42722.889 }, { "epoch": 1.2081584690280343, "grad_norm": 26.34969139099121, "learning_rate": 9.354562330429816e-05, "loss": 23.130556640625, "num_input_tokens_seen": 294769664, "step": 900, "train_runtime": 6898.4036, "train_tokens_per_second": 42730.127 }, { "epoch": 1.275306362262884, "grad_norm": 40.12674331665039, "learning_rate": 9.283164358132229e-05, "loss": 23.02666015625, "num_input_tokens_seen": 311153664, "step": 950, "train_runtime": 7280.3114, "train_tokens_per_second": 42739.06 }, { "epoch": 1.3424542554977337, "grad_norm": 26.551250457763672, "learning_rate": 9.211766385834643e-05, "loss": 22.74733642578125, "num_input_tokens_seen": 327537664, "step": 1000, "train_runtime": 7662.2497, "train_tokens_per_second": 42746.932 }, { "epoch": 1.3424542554977337, "eval_loss": 1.4059845209121704, "eval_masked_accuracy": 0.7082938621956478, "eval_runtime": 18.0953, "eval_samples_per_second": 217.349, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 327537664, "step": 1000 }, { "epoch": 1.4096021487325836, "grad_norm": 26.1070556640625, "learning_rate": 9.140368413537057e-05, "loss": 22.5372021484375, "num_input_tokens_seen": 343921664, "step": 1050, "train_runtime": 8065.2674, "train_tokens_per_second": 42642.314 }, { "epoch": 1.4767500419674333, "grad_norm": 26.403989791870117, "learning_rate": 9.068970441239469e-05, "loss": 22.54834228515625, "num_input_tokens_seen": 360305664, "step": 1100, "train_runtime": 8447.5369, "train_tokens_per_second": 42652.157 }, { "epoch": 1.543897935202283, "grad_norm": 27.94965362548828, "learning_rate": 8.997572468941883e-05, "loss": 22.2678564453125, "num_input_tokens_seen": 376689664, "step": 1150, "train_runtime": 8829.7028, "train_tokens_per_second": 42661.647 }, { "epoch": 1.6110458284371327, "grad_norm": 28.19874382019043, "learning_rate": 8.926174496644296e-05, "loss": 22.1563134765625, "num_input_tokens_seen": 393073664, "step": 1200, "train_runtime": 9212.0375, "train_tokens_per_second": 42669.568 }, { "epoch": 1.6781937216719824, "grad_norm": 23.451824188232422, "learning_rate": 8.854776524346708e-05, "loss": 21.9630419921875, "num_input_tokens_seen": 409457664, "step": 1250, "train_runtime": 9594.0382, "train_tokens_per_second": 42678.344 }, { "epoch": 1.7453416149068324, "grad_norm": 29.39751434326172, "learning_rate": 8.783378552049122e-05, "loss": 21.88580322265625, "num_input_tokens_seen": 425841664, "step": 1300, "train_runtime": 9976.9812, "train_tokens_per_second": 42682.416 }, { "epoch": 1.812489508141682, "grad_norm": 24.062496185302734, "learning_rate": 8.711980579751535e-05, "loss": 21.69278564453125, "num_input_tokens_seen": 442225664, "step": 1350, "train_runtime": 10359.0967, "train_tokens_per_second": 42689.597 }, { "epoch": 1.8796374013765318, "grad_norm": 31.466777801513672, "learning_rate": 8.640582607453949e-05, "loss": 21.459140625, "num_input_tokens_seen": 458609664, "step": 1400, "train_runtime": 10740.7818, "train_tokens_per_second": 42697.978 }, { "epoch": 1.9467852946113817, "grad_norm": 26.98602294921875, "learning_rate": 8.569184635156362e-05, "loss": 21.45286376953125, "num_input_tokens_seen": 474993664, "step": 1450, "train_runtime": 11123.0693, "train_tokens_per_second": 42703.471 }, { "epoch": 2.01342957864697, "grad_norm": 28.346364974975586, "learning_rate": 8.497786662858775e-05, "loss": 21.163310546875, "num_input_tokens_seen": 491235328, "step": 1500, "train_runtime": 11502.4827, "train_tokens_per_second": 42706.896 }, { "epoch": 2.01342957864697, "eval_loss": 1.322235345840454, "eval_masked_accuracy": 0.7215658834545166, "eval_runtime": 18.098, "eval_samples_per_second": 217.316, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 491235328, "step": 1500 }, { "epoch": 2.0805774718818197, "grad_norm": 24.22477149963379, "learning_rate": 8.426388690561189e-05, "loss": 21.14776123046875, "num_input_tokens_seen": 507619328, "step": 1550, "train_runtime": 11904.9842, "train_tokens_per_second": 42639.227 }, { "epoch": 2.1477253651166697, "grad_norm": 26.27544593811035, "learning_rate": 8.354990718263603e-05, "loss": 20.9605078125, "num_input_tokens_seen": 524003328, "step": 1600, "train_runtime": 12286.9253, "train_tokens_per_second": 42647.23 }, { "epoch": 2.214873258351519, "grad_norm": 28.20328712463379, "learning_rate": 8.283592745966014e-05, "loss": 20.92798095703125, "num_input_tokens_seen": 540387328, "step": 1650, "train_runtime": 12669.3045, "train_tokens_per_second": 42653.275 }, { "epoch": 2.282021151586369, "grad_norm": 23.897136688232422, "learning_rate": 8.212194773668428e-05, "loss": 20.790810546875, "num_input_tokens_seen": 556771328, "step": 1700, "train_runtime": 13051.1941, "train_tokens_per_second": 42660.566 }, { "epoch": 2.3491690448212186, "grad_norm": 27.00596809387207, "learning_rate": 8.140796801370842e-05, "loss": 20.76756591796875, "num_input_tokens_seen": 573155328, "step": 1750, "train_runtime": 13433.2554, "train_tokens_per_second": 42666.897 }, { "epoch": 2.4163169380560685, "grad_norm": 34.40134811401367, "learning_rate": 8.069398829073254e-05, "loss": 20.6015283203125, "num_input_tokens_seen": 589539328, "step": 1800, "train_runtime": 13814.9757, "train_tokens_per_second": 42673.932 }, { "epoch": 2.4834648312909184, "grad_norm": 31.715700149536133, "learning_rate": 7.998000856775668e-05, "loss": 20.6743505859375, "num_input_tokens_seen": 605923328, "step": 1850, "train_runtime": 14197.1991, "train_tokens_per_second": 42679.075 }, { "epoch": 2.550612724525768, "grad_norm": 23.027774810791016, "learning_rate": 7.926602884478082e-05, "loss": 20.44476318359375, "num_input_tokens_seen": 622307328, "step": 1900, "train_runtime": 14579.226, "train_tokens_per_second": 42684.525 }, { "epoch": 2.617760617760618, "grad_norm": 30.85573959350586, "learning_rate": 7.855204912180495e-05, "loss": 20.362332763671876, "num_input_tokens_seen": 638691328, "step": 1950, "train_runtime": 14961.5926, "train_tokens_per_second": 42688.726 }, { "epoch": 2.6849085109954673, "grad_norm": 24.106107711791992, "learning_rate": 7.783806939882907e-05, "loss": 20.340782470703125, "num_input_tokens_seen": 655075328, "step": 2000, "train_runtime": 15343.7785, "train_tokens_per_second": 42693.221 }, { "epoch": 2.6849085109954673, "eval_loss": 1.2741047143936157, "eval_masked_accuracy": 0.7301424206751543, "eval_runtime": 18.0943, "eval_samples_per_second": 217.361, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 655075328, "step": 2000 }, { "epoch": 2.7520564042303173, "grad_norm": 20.974048614501953, "learning_rate": 7.712408967585321e-05, "loss": 20.1228564453125, "num_input_tokens_seen": 671459328, "step": 2050, "train_runtime": 15745.9366, "train_tokens_per_second": 42643.34 }, { "epoch": 2.819204297465167, "grad_norm": 30.808757781982422, "learning_rate": 7.641010995287734e-05, "loss": 20.217064208984375, "num_input_tokens_seen": 687843328, "step": 2100, "train_runtime": 16128.0983, "train_tokens_per_second": 42648.756 }, { "epoch": 2.8863521907000167, "grad_norm": 23.832176208496094, "learning_rate": 7.569613022990148e-05, "loss": 20.1458984375, "num_input_tokens_seen": 704227328, "step": 2150, "train_runtime": 16510.4238, "train_tokens_per_second": 42653.498 }, { "epoch": 2.9535000839348666, "grad_norm": 88.81837463378906, "learning_rate": 7.49821505069256e-05, "loss": 19.993797607421875, "num_input_tokens_seen": 720611328, "step": 2200, "train_runtime": 16893.09, "train_tokens_per_second": 42657.165 }, { "epoch": 3.0201443679704547, "grad_norm": 25.890085220336914, "learning_rate": 7.426817078394974e-05, "loss": 19.76076171875, "num_input_tokens_seen": 736852992, "step": 2250, "train_runtime": 17272.5849, "train_tokens_per_second": 42660.262 }, { "epoch": 3.0872922612053046, "grad_norm": 21.964996337890625, "learning_rate": 7.355419106097388e-05, "loss": 19.6687451171875, "num_input_tokens_seen": 753236992, "step": 2300, "train_runtime": 17654.8634, "train_tokens_per_second": 42664.561 }, { "epoch": 3.1544401544401546, "grad_norm": 25.932247161865234, "learning_rate": 7.2840211337998e-05, "loss": 19.772432861328124, "num_input_tokens_seen": 769620992, "step": 2350, "train_runtime": 18036.9121, "train_tokens_per_second": 42669.221 }, { "epoch": 3.221588047675004, "grad_norm": 21.372344970703125, "learning_rate": 7.212623161502213e-05, "loss": 19.579937744140626, "num_input_tokens_seen": 786004992, "step": 2400, "train_runtime": 18418.9135, "train_tokens_per_second": 42673.798 }, { "epoch": 3.288735940909854, "grad_norm": 41.207489013671875, "learning_rate": 7.141225189204627e-05, "loss": 19.558551025390624, "num_input_tokens_seen": 802388992, "step": 2450, "train_runtime": 18801.1902, "train_tokens_per_second": 42677.564 }, { "epoch": 3.355883834144704, "grad_norm": 20.699298858642578, "learning_rate": 7.06982721690704e-05, "loss": 19.5032666015625, "num_input_tokens_seen": 818772992, "step": 2500, "train_runtime": 19183.4239, "train_tokens_per_second": 42681.275 }, { "epoch": 3.355883834144704, "eval_loss": 1.23164963722229, "eval_masked_accuracy": 0.7365415684330173, "eval_runtime": 18.1012, "eval_samples_per_second": 217.278, "eval_steps_per_second": 3.425, "num_input_tokens_seen": 818772992, "step": 2500 }, { "epoch": 3.4230317273795534, "grad_norm": 35.69466781616211, "learning_rate": 6.998429244609453e-05, "loss": 19.529766845703126, "num_input_tokens_seen": 835156992, "step": 2550, "train_runtime": 19586.4251, "train_tokens_per_second": 42639.583 }, { "epoch": 3.4901796206144033, "grad_norm": 22.10000991821289, "learning_rate": 6.927031272311867e-05, "loss": 19.36791015625, "num_input_tokens_seen": 851540992, "step": 2600, "train_runtime": 19967.8987, "train_tokens_per_second": 42645.498 }, { "epoch": 3.557327513849253, "grad_norm": 22.041030883789062, "learning_rate": 6.85563330001428e-05, "loss": 19.318353271484376, "num_input_tokens_seen": 867924992, "step": 2650, "train_runtime": 20350.1026, "train_tokens_per_second": 42649.662 }, { "epoch": 3.6244754070841028, "grad_norm": 23.8006591796875, "learning_rate": 6.784235327716694e-05, "loss": 19.310184326171875, "num_input_tokens_seen": 884308992, "step": 2700, "train_runtime": 20732.93, "train_tokens_per_second": 42652.389 }, { "epoch": 3.6916233003189527, "grad_norm": 21.48434066772461, "learning_rate": 6.712837355419106e-05, "loss": 19.189322509765624, "num_input_tokens_seen": 900692992, "step": 2750, "train_runtime": 21115.2157, "train_tokens_per_second": 42656.111 }, { "epoch": 3.758771193553802, "grad_norm": 27.336259841918945, "learning_rate": 6.641439383121519e-05, "loss": 19.158172607421875, "num_input_tokens_seen": 917076992, "step": 2800, "train_runtime": 21497.1386, "train_tokens_per_second": 42660.421 }, { "epoch": 3.825919086788652, "grad_norm": 20.084529876708984, "learning_rate": 6.570041410823933e-05, "loss": 19.0932373046875, "num_input_tokens_seen": 933460992, "step": 2850, "train_runtime": 21879.1616, "train_tokens_per_second": 42664.386 }, { "epoch": 3.8930669800235016, "grad_norm": 23.61543846130371, "learning_rate": 6.498643438526347e-05, "loss": 19.03164794921875, "num_input_tokens_seen": 949844992, "step": 2900, "train_runtime": 22261.824, "train_tokens_per_second": 42666.989 }, { "epoch": 3.9602148732583515, "grad_norm": 24.03072738647461, "learning_rate": 6.427245466228759e-05, "loss": 19.039158935546876, "num_input_tokens_seen": 966228992, "step": 2950, "train_runtime": 22643.7405, "train_tokens_per_second": 42670.909 }, { "epoch": 4.02685915729394, "grad_norm": 19.51736068725586, "learning_rate": 6.355847493931173e-05, "loss": 18.874622802734375, "num_input_tokens_seen": 982470656, "step": 3000, "train_runtime": 23023.9109, "train_tokens_per_second": 42671.754 }, { "epoch": 4.02685915729394, "eval_loss": 1.1917675733566284, "eval_masked_accuracy": 0.7423570262408483, "eval_runtime": 18.1046, "eval_samples_per_second": 217.238, "eval_steps_per_second": 3.425, "num_input_tokens_seen": 982470656, "step": 3000 }, { "epoch": 4.0940070505287895, "grad_norm": 22.008718490600586, "learning_rate": 6.284449521633586e-05, "loss": 18.817777099609376, "num_input_tokens_seen": 998854656, "step": 3050, "train_runtime": 23426.7943, "train_tokens_per_second": 42637.274 }, { "epoch": 4.1611549437636395, "grad_norm": 25.215585708618164, "learning_rate": 6.213051549335998e-05, "loss": 18.789967041015625, "num_input_tokens_seen": 1015238656, "step": 3100, "train_runtime": 23809.1884, "train_tokens_per_second": 42640.624 }, { "epoch": 4.228302836998489, "grad_norm": 24.252426147460938, "learning_rate": 6.141653577038412e-05, "loss": 18.7561767578125, "num_input_tokens_seen": 1031622656, "step": 3150, "train_runtime": 24191.755, "train_tokens_per_second": 42643.564 }, { "epoch": 4.295450730233339, "grad_norm": 21.702259063720703, "learning_rate": 6.0702556047408255e-05, "loss": 18.75292724609375, "num_input_tokens_seen": 1048006656, "step": 3200, "train_runtime": 24573.2637, "train_tokens_per_second": 42648.248 }, { "epoch": 4.362598623468188, "grad_norm": 95.9570541381836, "learning_rate": 5.998857632443239e-05, "loss": 18.597052001953124, "num_input_tokens_seen": 1064390656, "step": 3250, "train_runtime": 24955.5038, "train_tokens_per_second": 42651.54 }, { "epoch": 4.429746516703038, "grad_norm": 20.15200424194336, "learning_rate": 5.927459660145652e-05, "loss": 18.606531982421874, "num_input_tokens_seen": 1080774656, "step": 3300, "train_runtime": 25337.6615, "train_tokens_per_second": 42654.87 }, { "epoch": 4.496894409937888, "grad_norm": 20.575227737426758, "learning_rate": 5.856061687848066e-05, "loss": 18.473587646484376, "num_input_tokens_seen": 1097158656, "step": 3350, "train_runtime": 25719.2745, "train_tokens_per_second": 42659.005 }, { "epoch": 4.564042303172738, "grad_norm": 20.6302433013916, "learning_rate": 5.784663715550479e-05, "loss": 18.556181640625, "num_input_tokens_seen": 1113542656, "step": 3400, "train_runtime": 26100.9704, "train_tokens_per_second": 42662.883 }, { "epoch": 4.631190196407587, "grad_norm": 21.177248001098633, "learning_rate": 5.713265743252892e-05, "loss": 18.51771484375, "num_input_tokens_seen": 1129926656, "step": 3450, "train_runtime": 26483.1367, "train_tokens_per_second": 42665.892 }, { "epoch": 4.698338089642437, "grad_norm": 20.200031280517578, "learning_rate": 5.641867770955305e-05, "loss": 18.4009619140625, "num_input_tokens_seen": 1146310656, "step": 3500, "train_runtime": 26865.5394, "train_tokens_per_second": 42668.44 }, { "epoch": 4.698338089642437, "eval_loss": 1.1745275259017944, "eval_masked_accuracy": 0.7461218274111675, "eval_runtime": 18.093, "eval_samples_per_second": 217.376, "eval_steps_per_second": 3.427, "num_input_tokens_seen": 1146310656, "step": 3500 }, { "epoch": 4.765485982877287, "grad_norm": 22.780622482299805, "learning_rate": 5.570469798657718e-05, "loss": 18.44864013671875, "num_input_tokens_seen": 1162694656, "step": 3550, "train_runtime": 27267.9686, "train_tokens_per_second": 42639.577 }, { "epoch": 4.832633876112137, "grad_norm": 22.299123764038086, "learning_rate": 5.499071826360131e-05, "loss": 18.365985107421874, "num_input_tokens_seen": 1179078656, "step": 3600, "train_runtime": 27649.4052, "train_tokens_per_second": 42643.907 }, { "epoch": 4.899781769346987, "grad_norm": 20.132230758666992, "learning_rate": 5.427673854062545e-05, "loss": 18.296614990234374, "num_input_tokens_seen": 1195462656, "step": 3650, "train_runtime": 28031.8042, "train_tokens_per_second": 42646.654 }, { "epoch": 4.966929662581837, "grad_norm": 22.012845993041992, "learning_rate": 5.3562758817649584e-05, "loss": 18.1852392578125, "num_input_tokens_seen": 1211846656, "step": 3700, "train_runtime": 28413.9026, "train_tokens_per_second": 42649.779 }, { "epoch": 5.033573946617425, "grad_norm": 19.76921844482422, "learning_rate": 5.284877909467372e-05, "loss": 17.99932373046875, "num_input_tokens_seen": 1228088320, "step": 3750, "train_runtime": 28793.0555, "train_tokens_per_second": 42652.24 }, { "epoch": 5.100721839852275, "grad_norm": 19.71321678161621, "learning_rate": 5.213479937169785e-05, "loss": 18.186424560546875, "num_input_tokens_seen": 1244472320, "step": 3800, "train_runtime": 29175.5764, "train_tokens_per_second": 42654.592 }, { "epoch": 5.167869733087125, "grad_norm": 19.083816528320312, "learning_rate": 5.1420819648721974e-05, "loss": 18.081846923828124, "num_input_tokens_seen": 1260856320, "step": 3850, "train_runtime": 29557.9804, "train_tokens_per_second": 42657.052 }, { "epoch": 5.235017626321974, "grad_norm": 21.77515983581543, "learning_rate": 5.070683992574611e-05, "loss": 18.0148828125, "num_input_tokens_seen": 1277240320, "step": 3900, "train_runtime": 29940.6282, "train_tokens_per_second": 42659.102 }, { "epoch": 5.302165519556824, "grad_norm": 25.122718811035156, "learning_rate": 4.999286020277024e-05, "loss": 17.95366943359375, "num_input_tokens_seen": 1293624320, "step": 3950, "train_runtime": 30322.7025, "train_tokens_per_second": 42661.907 }, { "epoch": 5.369313412791674, "grad_norm": 20.051774978637695, "learning_rate": 4.927888047979438e-05, "loss": 17.99229736328125, "num_input_tokens_seen": 1310008320, "step": 4000, "train_runtime": 30705.0003, "train_tokens_per_second": 42664.332 }, { "epoch": 5.369313412791674, "eval_loss": 1.1476529836654663, "eval_masked_accuracy": 0.7508973219435078, "eval_runtime": 18.0946, "eval_samples_per_second": 217.358, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 1310008320, "step": 4000 }, { "epoch": 5.436461306026524, "grad_norm": 22.737159729003906, "learning_rate": 4.856490075681851e-05, "loss": 17.99468017578125, "num_input_tokens_seen": 1326392320, "step": 4050, "train_runtime": 31107.2176, "train_tokens_per_second": 42639.375 }, { "epoch": 5.503609199261373, "grad_norm": 19.653364181518555, "learning_rate": 4.785092103384264e-05, "loss": 17.9478125, "num_input_tokens_seen": 1342776320, "step": 4100, "train_runtime": 31489.8056, "train_tokens_per_second": 42641.62 }, { "epoch": 5.570757092496223, "grad_norm": 21.537843704223633, "learning_rate": 4.7136941310866775e-05, "loss": 17.9237060546875, "num_input_tokens_seen": 1359160320, "step": 4150, "train_runtime": 31871.5203, "train_tokens_per_second": 42644.979 }, { "epoch": 5.637904985731073, "grad_norm": 23.525545120239258, "learning_rate": 4.642296158789091e-05, "loss": 17.881396484375, "num_input_tokens_seen": 1375544320, "step": 4200, "train_runtime": 32253.1239, "train_tokens_per_second": 42648.406 }, { "epoch": 5.7050528789659225, "grad_norm": 27.282806396484375, "learning_rate": 4.570898186491504e-05, "loss": 17.892135009765624, "num_input_tokens_seen": 1391928320, "step": 4250, "train_runtime": 32635.0417, "train_tokens_per_second": 42651.342 }, { "epoch": 5.772200772200772, "grad_norm": 20.510400772094727, "learning_rate": 4.499500214193917e-05, "loss": 17.907117919921873, "num_input_tokens_seen": 1408312320, "step": 4300, "train_runtime": 33017.8689, "train_tokens_per_second": 42653.035 }, { "epoch": 5.839348665435622, "grad_norm": 19.230182647705078, "learning_rate": 4.4281022418963304e-05, "loss": 17.78746826171875, "num_input_tokens_seen": 1424696320, "step": 4350, "train_runtime": 33400.5094, "train_tokens_per_second": 42654.928 }, { "epoch": 5.906496558670471, "grad_norm": 24.074186325073242, "learning_rate": 4.3567042695987436e-05, "loss": 17.6533203125, "num_input_tokens_seen": 1441080320, "step": 4400, "train_runtime": 33783.0047, "train_tokens_per_second": 42656.961 }, { "epoch": 5.973644451905321, "grad_norm": 19.16803741455078, "learning_rate": 4.285306297301157e-05, "loss": 17.756685791015624, "num_input_tokens_seen": 1457464320, "step": 4450, "train_runtime": 34165.0509, "train_tokens_per_second": 42659.51 }, { "epoch": 6.040288735940909, "grad_norm": 20.361373901367188, "learning_rate": 4.21390832500357e-05, "loss": 17.543341064453124, "num_input_tokens_seen": 1473705984, "step": 4500, "train_runtime": 34544.159, "train_tokens_per_second": 42661.51 }, { "epoch": 6.040288735940909, "eval_loss": 1.1311087608337402, "eval_masked_accuracy": 0.7534060731038085, "eval_runtime": 18.1085, "eval_samples_per_second": 217.191, "eval_steps_per_second": 3.424, "num_input_tokens_seen": 1473705984, "step": 4500 }, { "epoch": 6.107436629175759, "grad_norm": 22.37014389038086, "learning_rate": 4.142510352705983e-05, "loss": 17.574459228515625, "num_input_tokens_seen": 1490089984, "step": 4550, "train_runtime": 34947.0679, "train_tokens_per_second": 42638.484 }, { "epoch": 6.174584522410609, "grad_norm": 40.5280647277832, "learning_rate": 4.0711123804083966e-05, "loss": 17.67244140625, "num_input_tokens_seen": 1506473984, "step": 4600, "train_runtime": 35329.4518, "train_tokens_per_second": 42640.74 }, { "epoch": 6.241732415645459, "grad_norm": 20.747886657714844, "learning_rate": 3.99971440811081e-05, "loss": 17.596712646484374, "num_input_tokens_seen": 1522857984, "step": 4650, "train_runtime": 35711.6986, "train_tokens_per_second": 42643.113 }, { "epoch": 6.308880308880309, "grad_norm": 24.38255500793457, "learning_rate": 3.928316435813223e-05, "loss": 17.52099853515625, "num_input_tokens_seen": 1539241984, "step": 4700, "train_runtime": 36093.7209, "train_tokens_per_second": 42645.7 }, { "epoch": 6.376028202115158, "grad_norm": 19.407743453979492, "learning_rate": 3.856918463515636e-05, "loss": 17.53602783203125, "num_input_tokens_seen": 1555625984, "step": 4750, "train_runtime": 36475.9888, "train_tokens_per_second": 42647.946 }, { "epoch": 6.443176095350008, "grad_norm": 20.32953453063965, "learning_rate": 3.7855204912180495e-05, "loss": 17.43239990234375, "num_input_tokens_seen": 1572009984, "step": 4800, "train_runtime": 36858.3509, "train_tokens_per_second": 42650.036 }, { "epoch": 6.510323988584858, "grad_norm": 19.538949966430664, "learning_rate": 3.714122518920463e-05, "loss": 17.46402099609375, "num_input_tokens_seen": 1588393984, "step": 4850, "train_runtime": 37240.3757, "train_tokens_per_second": 42652.469 }, { "epoch": 6.577471881819708, "grad_norm": 27.391788482666016, "learning_rate": 3.642724546622876e-05, "loss": 17.46167724609375, "num_input_tokens_seen": 1604777984, "step": 4900, "train_runtime": 37622.395, "train_tokens_per_second": 42654.86 }, { "epoch": 6.644619775054558, "grad_norm": 24.001676559448242, "learning_rate": 3.571326574325289e-05, "loss": 17.38479736328125, "num_input_tokens_seen": 1621161984, "step": 4950, "train_runtime": 38004.335, "train_tokens_per_second": 42657.291 }, { "epoch": 6.711767668289408, "grad_norm": 22.344423294067383, "learning_rate": 3.499928602027703e-05, "loss": 17.3615673828125, "num_input_tokens_seen": 1637545984, "step": 5000, "train_runtime": 38386.7223, "train_tokens_per_second": 42659.177 }, { "epoch": 6.711767668289408, "eval_loss": 1.1254918575286865, "eval_masked_accuracy": 0.7547708276205275, "eval_runtime": 18.0962, "eval_samples_per_second": 217.338, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 1637545984, "step": 5000 }, { "epoch": 6.778915561524257, "grad_norm": 20.349462509155273, "learning_rate": 3.4285306297301156e-05, "loss": 17.385616455078125, "num_input_tokens_seen": 1653929984, "step": 5050, "train_runtime": 38789.5078, "train_tokens_per_second": 42638.592 }, { "epoch": 6.846063454759107, "grad_norm": 19.716110229492188, "learning_rate": 3.357132657432529e-05, "loss": 17.39263671875, "num_input_tokens_seen": 1670313984, "step": 5100, "train_runtime": 39171.8181, "train_tokens_per_second": 42640.706 }, { "epoch": 6.913211347993957, "grad_norm": 23.193069458007812, "learning_rate": 3.285734685134942e-05, "loss": 17.322354736328126, "num_input_tokens_seen": 1686697984, "step": 5150, "train_runtime": 39553.757, "train_tokens_per_second": 42643.18 }, { "epoch": 6.980359241228807, "grad_norm": 19.377727508544922, "learning_rate": 3.214336712837356e-05, "loss": 17.24675048828125, "num_input_tokens_seen": 1703081984, "step": 5200, "train_runtime": 39936.046, "train_tokens_per_second": 42645.233 }, { "epoch": 7.047003525264395, "grad_norm": 19.60344696044922, "learning_rate": 3.1429387405397685e-05, "loss": 17.098548583984375, "num_input_tokens_seen": 1719323648, "step": 5250, "train_runtime": 40315.6311, "train_tokens_per_second": 42646.577 }, { "epoch": 7.114151418499245, "grad_norm": 22.405685424804688, "learning_rate": 3.071540768242182e-05, "loss": 17.122021484375, "num_input_tokens_seen": 1735707648, "step": 5300, "train_runtime": 40697.747, "train_tokens_per_second": 42648.74 }, { "epoch": 7.181299311734095, "grad_norm": 19.352375030517578, "learning_rate": 3.0001427959445953e-05, "loss": 17.219647216796876, "num_input_tokens_seen": 1752091648, "step": 5350, "train_runtime": 41080.3111, "train_tokens_per_second": 42650.399 }, { "epoch": 7.248447204968944, "grad_norm": 19.293392181396484, "learning_rate": 2.9287448236470082e-05, "loss": 17.0911181640625, "num_input_tokens_seen": 1768475648, "step": 5400, "train_runtime": 41463.0281, "train_tokens_per_second": 42651.869 }, { "epoch": 7.315595098203794, "grad_norm": 20.061555862426758, "learning_rate": 2.8573468513494218e-05, "loss": 17.16145751953125, "num_input_tokens_seen": 1784859648, "step": 5450, "train_runtime": 41845.4451, "train_tokens_per_second": 42653.618 }, { "epoch": 7.3827429914386435, "grad_norm": 19.52706527709961, "learning_rate": 2.785948879051835e-05, "loss": 17.160897216796876, "num_input_tokens_seen": 1801243648, "step": 5500, "train_runtime": 42228.0537, "train_tokens_per_second": 42655.143 }, { "epoch": 7.3827429914386435, "eval_loss": 1.1056840419769287, "eval_masked_accuracy": 0.7587028345843005, "eval_runtime": 18.1241, "eval_samples_per_second": 217.004, "eval_steps_per_second": 3.421, "num_input_tokens_seen": 1801243648, "step": 5500 }, { "epoch": 7.4498908846734935, "grad_norm": 18.919979095458984, "learning_rate": 2.7145509067542486e-05, "loss": 17.10321044921875, "num_input_tokens_seen": 1817627648, "step": 5550, "train_runtime": 42630.7742, "train_tokens_per_second": 42636.515 }, { "epoch": 7.517038777908343, "grad_norm": 19.892620086669922, "learning_rate": 2.6431529344566615e-05, "loss": 17.095526123046874, "num_input_tokens_seen": 1834011648, "step": 5600, "train_runtime": 43013.4145, "train_tokens_per_second": 42638.132 }, { "epoch": 7.584186671143193, "grad_norm": 20.655860900878906, "learning_rate": 2.5717549621590747e-05, "loss": 17.05685302734375, "num_input_tokens_seen": 1850395648, "step": 5650, "train_runtime": 43395.7235, "train_tokens_per_second": 42640.046 }, { "epoch": 7.651334564378042, "grad_norm": 19.32682228088379, "learning_rate": 2.5003569898614883e-05, "loss": 16.990732421875, "num_input_tokens_seen": 1866779648, "step": 5700, "train_runtime": 43777.9257, "train_tokens_per_second": 42642.031 }, { "epoch": 7.718482457612892, "grad_norm": 20.431968688964844, "learning_rate": 2.4289590175639012e-05, "loss": 17.013017578125, "num_input_tokens_seen": 1883163648, "step": 5750, "train_runtime": 44160.1064, "train_tokens_per_second": 42644.002 }, { "epoch": 7.785630350847742, "grad_norm": 24.700050354003906, "learning_rate": 2.3575610452663147e-05, "loss": 16.923485107421875, "num_input_tokens_seen": 1899547648, "step": 5800, "train_runtime": 44542.2389, "train_tokens_per_second": 42645.985 }, { "epoch": 7.852778244082592, "grad_norm": 26.109970092773438, "learning_rate": 2.2861630729687276e-05, "loss": 16.96089111328125, "num_input_tokens_seen": 1915931648, "step": 5850, "train_runtime": 44924.6097, "train_tokens_per_second": 42647.708 }, { "epoch": 7.919926137317441, "grad_norm": 40.166683197021484, "learning_rate": 2.2147651006711412e-05, "loss": 16.97099365234375, "num_input_tokens_seen": 1932315648, "step": 5900, "train_runtime": 45306.7483, "train_tokens_per_second": 42649.621 }, { "epoch": 7.987074030552291, "grad_norm": 20.37265968322754, "learning_rate": 2.1433671283735544e-05, "loss": 16.91640625, "num_input_tokens_seen": 1948699648, "step": 5950, "train_runtime": 45688.7131, "train_tokens_per_second": 42651.664 }, { "epoch": 8.05371831458788, "grad_norm": 20.146041870117188, "learning_rate": 2.0719691560759673e-05, "loss": 16.76911865234375, "num_input_tokens_seen": 1964941312, "step": 6000, "train_runtime": 46067.8327, "train_tokens_per_second": 42653.218 }, { "epoch": 8.05371831458788, "eval_loss": 1.096535086631775, "eval_masked_accuracy": 0.7597164266773505, "eval_runtime": 18.1153, "eval_samples_per_second": 217.109, "eval_steps_per_second": 3.423, "num_input_tokens_seen": 1964941312, "step": 6000 }, { "epoch": 8.12086620782273, "grad_norm": 19.625225067138672, "learning_rate": 2.000571183778381e-05, "loss": 16.9940234375, "num_input_tokens_seen": 1981325312, "step": 6050, "train_runtime": 46469.6541, "train_tokens_per_second": 42636.971 }, { "epoch": 8.188014101057579, "grad_norm": 18.703025817871094, "learning_rate": 1.9291732114807938e-05, "loss": 16.83005126953125, "num_input_tokens_seen": 1997709312, "step": 6100, "train_runtime": 46851.8703, "train_tokens_per_second": 42638.838 }, { "epoch": 8.255161994292429, "grad_norm": 20.429426193237305, "learning_rate": 1.8577752391832074e-05, "loss": 16.810699462890625, "num_input_tokens_seen": 2014093312, "step": 6150, "train_runtime": 47233.5489, "train_tokens_per_second": 42641.16 }, { "epoch": 8.322309887527279, "grad_norm": 21.28598976135254, "learning_rate": 1.7863772668856206e-05, "loss": 16.7576171875, "num_input_tokens_seen": 2030477312, "step": 6200, "train_runtime": 47616.4549, "train_tokens_per_second": 42642.345 }, { "epoch": 8.389457780762129, "grad_norm": 21.31905174255371, "learning_rate": 1.7149792945880338e-05, "loss": 16.805338134765623, "num_input_tokens_seen": 2046861312, "step": 6250, "train_runtime": 47999.4484, "train_tokens_per_second": 42643.434 }, { "epoch": 8.456605673996979, "grad_norm": 18.618684768676758, "learning_rate": 1.643581322290447e-05, "loss": 16.7966064453125, "num_input_tokens_seen": 2063245312, "step": 6300, "train_runtime": 48381.6311, "train_tokens_per_second": 42645.22 }, { "epoch": 8.523753567231829, "grad_norm": 21.598894119262695, "learning_rate": 1.5721833499928603e-05, "loss": 16.784290771484375, "num_input_tokens_seen": 2079629312, "step": 6350, "train_runtime": 48763.1337, "train_tokens_per_second": 42647.573 }, { "epoch": 8.590901460466679, "grad_norm": 30.116657257080078, "learning_rate": 1.5007853776952735e-05, "loss": 16.74375, "num_input_tokens_seen": 2096013312, "step": 6400, "train_runtime": 49145.7118, "train_tokens_per_second": 42648.956 }, { "epoch": 8.658049353701527, "grad_norm": 60.88018035888672, "learning_rate": 1.4293874053976869e-05, "loss": 16.743026123046874, "num_input_tokens_seen": 2112397312, "step": 6450, "train_runtime": 49527.861, "train_tokens_per_second": 42650.687 }, { "epoch": 8.725197246936377, "grad_norm": 79.8419189453125, "learning_rate": 1.3579894331001e-05, "loss": 16.7031982421875, "num_input_tokens_seen": 2128781312, "step": 6500, "train_runtime": 49910.3805, "train_tokens_per_second": 42652.075 }, { "epoch": 8.725197246936377, "eval_loss": 1.0834989547729492, "eval_masked_accuracy": 0.7624798575184463, "eval_runtime": 18.0968, "eval_samples_per_second": 217.331, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 2128781312, "step": 6500 }, { "epoch": 8.792345140171227, "grad_norm": 24.531356811523438, "learning_rate": 1.2865914608025132e-05, "loss": 16.679058837890626, "num_input_tokens_seen": 2145165312, "step": 6550, "train_runtime": 50313.0537, "train_tokens_per_second": 42636.357 }, { "epoch": 8.859493033406077, "grad_norm": 19.91719627380371, "learning_rate": 1.2151934885049266e-05, "loss": 16.6896630859375, "num_input_tokens_seen": 2161549312, "step": 6600, "train_runtime": 50694.9327, "train_tokens_per_second": 42638.37 }, { "epoch": 8.926640926640927, "grad_norm": 85.14508056640625, "learning_rate": 1.1437955162073398e-05, "loss": 16.694083251953124, "num_input_tokens_seen": 2177933312, "step": 6650, "train_runtime": 51077.1242, "train_tokens_per_second": 42640.093 }, { "epoch": 8.993788819875776, "grad_norm": 19.09105682373047, "learning_rate": 1.072397543909753e-05, "loss": 16.690823974609376, "num_input_tokens_seen": 2194317312, "step": 6700, "train_runtime": 51459.4495, "train_tokens_per_second": 42641.679 }, { "epoch": 9.060433103911365, "grad_norm": 20.286251068115234, "learning_rate": 1.0009995716121663e-05, "loss": 16.543135986328124, "num_input_tokens_seen": 2210558976, "step": 6750, "train_runtime": 51838.5574, "train_tokens_per_second": 42643.142 }, { "epoch": 9.127580997146215, "grad_norm": 19.63930892944336, "learning_rate": 9.296015993145795e-06, "loss": 16.60925048828125, "num_input_tokens_seen": 2226942976, "step": 6800, "train_runtime": 52221.018, "train_tokens_per_second": 42644.572 }, { "epoch": 9.194728890381064, "grad_norm": 20.226930618286133, "learning_rate": 8.582036270169927e-06, "loss": 16.68048828125, "num_input_tokens_seen": 2243326976, "step": 6850, "train_runtime": 52603.1483, "train_tokens_per_second": 42646.249 }, { "epoch": 9.261876783615914, "grad_norm": 18.24042320251465, "learning_rate": 7.86805654719406e-06, "loss": 16.55905517578125, "num_input_tokens_seen": 2259710976, "step": 6900, "train_runtime": 52985.937, "train_tokens_per_second": 42647.372 }, { "epoch": 9.329024676850764, "grad_norm": 22.2435245513916, "learning_rate": 7.154076824218192e-06, "loss": 16.5089697265625, "num_input_tokens_seen": 2276094976, "step": 6950, "train_runtime": 53367.9926, "train_tokens_per_second": 42649.065 }, { "epoch": 9.396172570085614, "grad_norm": 20.44721031188965, "learning_rate": 6.440097101242325e-06, "loss": 16.580111083984374, "num_input_tokens_seen": 2292478976, "step": 7000, "train_runtime": 53749.9804, "train_tokens_per_second": 42650.787 }, { "epoch": 9.396172570085614, "eval_loss": 1.0845201015472412, "eval_masked_accuracy": 0.7618803372351338, "eval_runtime": 18.1075, "eval_samples_per_second": 217.202, "eval_steps_per_second": 3.424, "num_input_tokens_seen": 2292478976, "step": 7000 }, { "epoch": 9.463320463320464, "grad_norm": 18.416465759277344, "learning_rate": 5.726117378266457e-06, "loss": 16.533978271484376, "num_input_tokens_seen": 2308862976, "step": 7050, "train_runtime": 54152.7328, "train_tokens_per_second": 42636.13 }, { "epoch": 9.530468356555312, "grad_norm": 19.039291381835938, "learning_rate": 5.01213765529059e-06, "loss": 16.6065771484375, "num_input_tokens_seen": 2325246976, "step": 7100, "train_runtime": 54534.7391, "train_tokens_per_second": 42637.904 }, { "epoch": 9.597616249790162, "grad_norm": 19.21045684814453, "learning_rate": 4.298157932314723e-06, "loss": 16.554276123046876, "num_input_tokens_seen": 2341630976, "step": 7150, "train_runtime": 54916.6532, "train_tokens_per_second": 42639.725 }, { "epoch": 9.664764143025012, "grad_norm": 21.09920310974121, "learning_rate": 3.584178209338855e-06, "loss": 16.55759033203125, "num_input_tokens_seen": 2358014976, "step": 7200, "train_runtime": 55298.628, "train_tokens_per_second": 42641.473 }, { "epoch": 9.731912036259862, "grad_norm": 30.43587875366211, "learning_rate": 2.8701984863629874e-06, "loss": 16.526624755859373, "num_input_tokens_seen": 2374398976, "step": 7250, "train_runtime": 55680.03, "train_tokens_per_second": 42643.637 }, { "epoch": 9.799059929494712, "grad_norm": 25.07560157775879, "learning_rate": 2.15621876338712e-06, "loss": 16.5307763671875, "num_input_tokens_seen": 2390782976, "step": 7300, "train_runtime": 56062.5396, "train_tokens_per_second": 42644.928 }, { "epoch": 9.866207822729562, "grad_norm": 24.447111129760742, "learning_rate": 1.4422390404112524e-06, "loss": 16.479052734375, "num_input_tokens_seen": 2407166976, "step": 7350, "train_runtime": 56444.4228, "train_tokens_per_second": 42646.675 }, { "epoch": 9.933355715964412, "grad_norm": 23.812557220458984, "learning_rate": 7.282593174353849e-07, "loss": 16.499632568359374, "num_input_tokens_seen": 2423550976, "step": 7400, "train_runtime": 56826.7541, "train_tokens_per_second": 42648.063 }, { "epoch": 10.0, "grad_norm": 27.18109130859375, "learning_rate": 1.427959445951735e-08, "loss": 16.4600341796875, "num_input_tokens_seen": 2439792640, "step": 7450, "train_runtime": 57205.5292, "train_tokens_per_second": 42649.595 }, { "epoch": 10.0, "eval_loss": 1.0779017210006714, "eval_masked_accuracy": 0.763301870184039, "eval_runtime": 18.0983, "eval_samples_per_second": 217.313, "eval_steps_per_second": 3.426, "num_input_tokens_seen": 2439792640, "step": 7450 }, { "epoch": 10.0, "num_input_tokens_seen": 2439792640, "step": 7450, "total_flos": 8.192069135329812e+18, "train_loss": 19.39006050468291, "train_runtime": 57225.9994, "train_samples_per_second": 83.27, "train_steps_per_second": 0.13 } ], "logging_steps": 50, "max_steps": 7450, "num_input_tokens_seen": 2439792640, "num_train_epochs": 10, "save_steps": 500, "stateful_callbacks": { "EarlyStoppingCallback": { "args": { "early_stopping_patience": 5, "early_stopping_threshold": 0.0001 }, "attributes": { "early_stopping_patience_counter": 0 } }, "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 8.192069135329812e+18, "train_batch_size": 40, "trial_name": null, "trial_params": null }