sweep/summary_lean.csv

tag,n_layer,n_embd,n_head,num_params,n_seqs,seq_len,vocab_size,dtype,seed,max_iters,batch_size,grad_accum,H_x_bits,H_x_given_theta_bits,mem_bits,bpp,train_loss,test_loss,mi_f1,mi_threshold,elapsed_s
d1-w32_n1024_bf16_s0,1,32,4,77920,1024,64,2048,bfloat16,0,6409,1024,1,720896.0,486211.69511610514,234684.30488389486,3.0118622290027575,5.142460107803345,10.926623582839966,0.9995119570517197,-9.591796875,161.96897435188293
d1-w32_n1024_fp32_s0,1,32,4,77920,1024,64,2048,float32,0,6000,512,1,720896.0,481075.79093898047,239820.20906101953,3.077774756943269,5.088139772415161,10.842220306396484,0.9990243902434016,-9.790265083312988,61.78060817718506
d1-w32_n16384_bf16_s0,1,32,4,77920,16384,64,2048,bfloat16,0,12553,1024,1,11534336.0,11258468.890324092,275867.10967590846,3.54038898454708,7.442260712385178,7.800113916397095,0.9974681999811978,-7.6513671875,316.7593641281128
d1-w32_n16384_fp32_s0,1,32,4,77920,16384,64,2048,float32,0,12000,512,1,11534336.0,11289668.252604596,244667.74739540368,3.139986491214113,7.4628846347332,7.7802863121032715,0.9958460598651078,-7.651047706604004,124.28364586830139
d1-w32_n4096_bf16_s0,1,32,4,77920,4096,64,2048,bfloat16,0,7638,1024,1,2883584.0,2623335.6634027935,260248.33659720654,3.3399427181366343,6.936484217643738,8.30504560470581,0.9998779445863422,-7.779296875,192.68513226509094
d2-w128_n1024_bfloat16_s0,2,128,8,656000,1024,64,2048,bfloat16,0,8000,384,1,720896.0,495.70091115032506,720400.2990888497,1.0981711876354416,0.005242823623120785,27.642335414886475,0.9999999999994991,-0.0300984475761652,167.94316267967224
d2-w128_n16384_bfloat16_s0,2,128,8,656000,16384,64,2048,bfloat16,0,28000,384,1,11534336.0,9164760.124239756,2369575.875760244,3.612158347195494,6.058242455124855,9.203715562820435,0.999816860996997,-6.941162109375,592.8328263759613
d2-w128_n4096_bfloat16_s0,2,128,8,656000,4096,64,2048,bfloat16,0,15000,384,1,2883584.0,634770.3991878944,2248813.6008121055,3.4280695134330874,1.6784260272979736,18.886457443237305,0.9998779445863422,-16.809814453125,319.55566334724426
d2-w64_n2048_bfloat16_s0,2,64,4,229696,2048,64,2048,bfloat16,0,10000,512,1,1441792.0,716151.5978994451,725640.4021005549,3.159133820791633,3.7872197031974792,13.130642890930176,0.9997559189645959,-11.4853515625,222.75749039649963
d2-w64_n2048_fp32_s0,2,64,4,229696,2048,64,2048,float32,0,10000,256,1,1441792.0,714860.3632957394,726931.6367042606,3.164755314434124,3.780391275882721,12.841822147369385,0.9990224828929501,-4.910470962524414,78.76775193214417
d2-w64_n32768_bfloat16_s0,2,64,4,229696,32768,64,2048,bfloat16,0,30000,512,1,23068672.0,22239409.105432916,829262.8945670836,3.610262671387763,7.3505323976278305,7.892160654067993,0.9996033320106067,-7.66845703125,670.8058476448059
d2-w64_n512_bfloat16_s0,2,64,4,229696,512,64,2048,bfloat16,0,5000,512,1,360448.0,193.7184181543366,360254.28158184566,1.568395973729824,0.004097759258002043,29.440959453582764,0.9990224828929486,-0.05276238173246384,112.13331604003906
d2-w64_n8192_bfloat16_s0,2,64,4,229696,8192,64,2048,bfloat16,0,22000,512,1,5767168.0,4944240.743331735,822927.256668265,3.582679962508119,6.536648809909821,8.729503870010376,0.9999999999994997,-7.302978515625,488.9944713115692
d2-w64_n8192_fp32_s0,2,64,4,229696,8192,64,2048,float32,0,22000,256,1,5767168.0,4952531.934273833,814636.0657261666,3.5465835962583876,6.547610372304916,8.696101903915405,0.999694731057562,-7.18479061126709,173.75297904014587
d4-w128_n2048_bfloat16_s0,4,128,8,1049728,2048,64,2048,bfloat16,0,10000,384,1,1441792.0,2184.859671600569,1439607.1403283994,1.3714096797726643,0.011554178781807423,26.50293493270874,0.9990224828929501,-0.05870996415615082,211.60499811172485
d4-w128_n32768_bfloat16_s0,4,128,8,1049728,32768,64,2048,bfloat16,0,32000,384,1,23068672.0,19322809.427256536,3745862.5727434643,3.568412553293295,6.386542737483978,8.832721948623657,0.9998931770661422,-7.30322265625,673.7350013256073
d4-w128_n8192_bfloat16_s0,4,128,8,1049728,8192,64,2048,bfloat16,0,22000,384,1,5767168.0,2019720.2974360613,3747447.7025639387,3.5699225919132753,2.6702183336019516,14.464654445648193,0.9998779147840194,-4.147796630859375,468.3837842941284