Upload folder using huggingface_hub

logs/fix_compile_bug/disable_dynamo/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "logs/fix_compile_bug/disable_dynamo", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 12, "n_embd": 264, "head_dim": 22, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 4375, "warmup_steps": 250, "group": "fix_compile_bug", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1337, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "disable_selection_head_linear_combo_bias": false, "assert_latent_matches_no_head": false, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 32, "total_batch_size": 131072, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 1, "init_latent_masks_to_identity": true, "latent_mask_scale": null, "latent_mask_sigmoid": false, "S_layernorm": false, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.003, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "disable_dynamo"}

logs/fix_compile_bug/disable_dynamo/dataloader_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6031fd3e2855a036f7a5531cc24555aabd1115f9dd6618b8b2ca6f55279ef0b2
+size 964

logs/fix_compile_bug/disable_dynamo/log2.txt

@@ -0,0 +1,529 @@
+max_steps: 4375
+0 val loss 11.2679
+0 val perplexity 78270.6641
+0 train 11.273981 (lr=4.1958e-06) (hash(x)=42405186)
+10 train 9.975639 (lr=4.6154e-05) (hash(x)=38970388)
+20 train 9.416135 (lr=8.8112e-05) (hash(x)=38844124)
+30 train 8.637611 (lr=1.3007e-04) (hash(x)=37597230)
+40 train 7.895756 (lr=1.7203e-04) (hash(x)=40798817)
+50 train 7.700012 (lr=2.1399e-04) (hash(x)=36405572)
+60 train 7.650867 (lr=2.5594e-04) (hash(x)=36066574)
+70 train 7.536690 (lr=2.9790e-04) (hash(x)=43811010)
+80 train 7.531866 (lr=3.3986e-04) (hash(x)=40628178)
+90 train 7.173455 (lr=3.8182e-04) (hash(x)=43681905)
+100 val loss 7.0869
+100 val perplexity 1196.2357
+100 train 6.970158 (lr=4.2378e-04) (hash(x)=42344017)
+110 train 7.016336 (lr=4.6573e-04) (hash(x)=39853582)
+120 train 6.827969 (lr=5.0769e-04) (hash(x)=39880304)
+130 train 6.595816 (lr=5.4965e-04) (hash(x)=36124250)
+140 train 6.529469 (lr=5.9161e-04) (hash(x)=45012802)
+150 train 6.381016 (lr=6.3357e-04) (hash(x)=36325989)
+160 train 6.290360 (lr=6.7552e-04) (hash(x)=40241802)
+170 train 6.293976 (lr=7.1748e-04) (hash(x)=42401819)
+180 train 6.203776 (lr=7.5944e-04) (hash(x)=40459165)
+190 train 6.184151 (lr=8.0140e-04) (hash(x)=41967324)
+200 val loss 6.1795
+200 val perplexity 482.7632
+200 train 6.227951 (lr=8.4336e-04) (hash(x)=45796175)
+210 train 6.128983 (lr=8.8531e-04) (hash(x)=35252224)
+220 train 6.038887 (lr=9.2727e-04) (hash(x)=40863261)
+230 train 5.878501 (lr=9.6923e-04) (hash(x)=40738876)
+240 train 5.875349 (lr=1.0112e-03) (hash(x)=46763138)
+250 train 5.924514 (lr=1.0531e-03) (hash(x)=46920901)
+260 train 5.706999 (lr=1.0951e-03) (hash(x)=43475159)
+270 train 5.721182 (lr=1.1371e-03) (hash(x)=47424242)
+280 train 5.694764 (lr=1.1790e-03) (hash(x)=40412635)
+290 train 5.621622 (lr=1.2210e-03) (hash(x)=31782544)
+300 val loss 5.6890
+300 val perplexity 295.5923
+300 train 5.579260 (lr=1.2629e-03) (hash(x)=34812634)
+310 train 5.493578 (lr=1.3049e-03) (hash(x)=44178651)
+320 train 5.656520 (lr=1.3469e-03) (hash(x)=43205331)
+330 train 5.600074 (lr=1.3888e-03) (hash(x)=41565398)
+340 train 5.618014 (lr=1.4308e-03) (hash(x)=41719738)
+350 train 5.468098 (lr=1.4727e-03) (hash(x)=46108058)
+360 train 5.431486 (lr=1.5147e-03) (hash(x)=37445628)
+370 train 5.313436 (lr=1.5566e-03) (hash(x)=36085988)
+380 train 5.382994 (lr=1.5986e-03) (hash(x)=37645228)
+390 train 5.382205 (lr=1.6406e-03) (hash(x)=35483844)
+400 val loss 5.3499
+400 val perplexity 210.5906
+400 train 5.355009 (lr=1.6825e-03) (hash(x)=41699751)
+410 train 5.197707 (lr=1.7245e-03) (hash(x)=32755762)
+420 train 5.192819 (lr=1.7664e-03) (hash(x)=35953145)
+430 train 5.181862 (lr=1.8084e-03) (hash(x)=35587716)
+440 train 5.127245 (lr=1.8503e-03) (hash(x)=39335888)
+450 train 5.163924 (lr=1.8923e-03) (hash(x)=44314559)
+460 train 5.238398 (lr=1.9343e-03) (hash(x)=42950490)
+470 train 5.305055 (lr=1.9762e-03) (hash(x)=41461061)
+480 train 5.228039 (lr=2.0182e-03) (hash(x)=41993931)
+490 train 5.143535 (lr=2.0601e-03) (hash(x)=37544536)
+500 val loss 5.1118
+500 val perplexity 165.9736
+500 train 5.105995 (lr=2.1021e-03) (hash(x)=40203267)
+510 train 5.111572 (lr=2.1441e-03) (hash(x)=41388334)
+520 train 5.063201 (lr=2.1860e-03) (hash(x)=34662125)
+530 train 5.142380 (lr=2.2280e-03) (hash(x)=37501995)
+540 train 5.057854 (lr=2.2699e-03) (hash(x)=41263260)
+550 train 4.970294 (lr=2.3119e-03) (hash(x)=49637102)
+560 train 4.888000 (lr=2.3538e-03) (hash(x)=47434359)
+570 train 4.858415 (lr=2.3958e-03) (hash(x)=42912781)
+580 train 4.932686 (lr=2.4378e-03) (hash(x)=48538862)
+590 train 4.779510 (lr=2.4797e-03) (hash(x)=38595222)
+600 val loss 4.9338
+600 val perplexity 138.9072
+600 train 4.868814 (lr=2.5217e-03) (hash(x)=37146098)
+610 train 4.907523 (lr=2.5636e-03) (hash(x)=50116590)
+620 train 4.870512 (lr=2.6056e-03) (hash(x)=41832213)
+630 train 4.979991 (lr=2.6476e-03) (hash(x)=39030069)
+640 train 4.832273 (lr=2.6895e-03) (hash(x)=46237222)
+650 train 4.845035 (lr=2.7315e-03) (hash(x)=42787835)
+660 train 4.755742 (lr=2.7734e-03) (hash(x)=39109857)
+670 train 4.849321 (lr=2.8154e-03) (hash(x)=39462485)
+680 train 4.739604 (lr=2.8573e-03) (hash(x)=41744849)
+690 train 4.757978 (lr=2.8993e-03) (hash(x)=38408104)
+700 val loss 4.7751
+700 val perplexity 118.5169
+700 train 4.661257 (lr=2.9413e-03) (hash(x)=41890521)
+710 train 4.725952 (lr=2.9832e-03) (hash(x)=41416330)
+720 train 4.627008 (lr=3.0000e-03) (hash(x)=38760874)
+730 train 4.717939 (lr=2.9999e-03) (hash(x)=37451366)
+740 train 4.613894 (lr=2.9997e-03) (hash(x)=36120996)
+750 train 4.714878 (lr=2.9994e-03) (hash(x)=34667979)
+760 train 4.708947 (lr=2.9990e-03) (hash(x)=50862178)
+770 train 4.653719 (lr=2.9985e-03) (hash(x)=41230008)
+780 train 4.555442 (lr=2.9979e-03) (hash(x)=36407773)
+790 train 4.564205 (lr=2.9972e-03) (hash(x)=42561722)
+800 val loss 4.6624
+800 val perplexity 105.8869
+800 train 4.539184 (lr=2.9964e-03) (hash(x)=48340441)
+810 train 4.608728 (lr=2.9955e-03) (hash(x)=36261826)
+820 train 4.426336 (lr=2.9945e-03) (hash(x)=37918881)
+830 train 4.543477 (lr=2.9934e-03) (hash(x)=38608334)
+840 train 4.282201 (lr=2.9922e-03) (hash(x)=39403505)
+850 train 4.398445 (lr=2.9909e-03) (hash(x)=43741441)
+860 train 4.351716 (lr=2.9896e-03) (hash(x)=38030739)
+870 train 4.652875 (lr=2.9881e-03) (hash(x)=36064710)
+880 train 4.812729 (lr=2.9865e-03) (hash(x)=40349968)
+890 train 4.567600 (lr=2.9848e-03) (hash(x)=38715806)
+900 val loss 4.5712
+900 val perplexity 96.6582
+900 train 4.633204 (lr=2.9830e-03) (hash(x)=35310433)
+910 train 4.521075 (lr=2.9811e-03) (hash(x)=37426246)
+920 train 4.507960 (lr=2.9792e-03) (hash(x)=39966506)
+930 train 4.523317 (lr=2.9771e-03) (hash(x)=37747046)
+940 train 4.478729 (lr=2.9749e-03) (hash(x)=42710584)
+950 train 4.501898 (lr=2.9726e-03) (hash(x)=41617662)
+960 train 4.497109 (lr=2.9703e-03) (hash(x)=40145473)
+970 train 4.485673 (lr=2.9678e-03) (hash(x)=38513059)
+980 train 4.486294 (lr=2.9652e-03) (hash(x)=38574237)
+990 train 4.401441 (lr=2.9626e-03) (hash(x)=37633181)
+1000 val loss 4.5003
+1000 val perplexity 90.0464
+1000 train 4.278323 (lr=2.9598e-03) (hash(x)=42781027)
+1010 train 4.420980 (lr=2.9570e-03) (hash(x)=48329642)
+1020 train 4.287330 (lr=2.9540e-03) (hash(x)=36577969)
+1030 train 4.257991 (lr=2.9510e-03) (hash(x)=39789757)
+1040 train 4.196259 (lr=2.9478e-03) (hash(x)=43293764)
+1050 train 4.534105 (lr=2.9446e-03) (hash(x)=40077223)
+1060 train 4.544160 (lr=2.9412e-03) (hash(x)=41035822)
+1070 train 4.456591 (lr=2.9378e-03) (hash(x)=42069615)
+1080 train 4.455237 (lr=2.9343e-03) (hash(x)=42668159)
+1090 train 4.429324 (lr=2.9307e-03) (hash(x)=36477372)
+1100 val loss 4.4191
+1100 val perplexity 83.0176
+1100 train 4.482910 (lr=2.9270e-03) (hash(x)=41357005)
+1110 train 4.439171 (lr=2.9231e-03) (hash(x)=42852875)
+1120 train 4.483402 (lr=2.9192e-03) (hash(x)=35310013)
+1130 train 4.478104 (lr=2.9152e-03) (hash(x)=51231839)
+1140 train 4.419302 (lr=2.9112e-03) (hash(x)=41431568)
+1150 train 4.400342 (lr=2.9070e-03) (hash(x)=41848748)
+1160 train 4.334244 (lr=2.9027e-03) (hash(x)=36702617)
+1170 train 4.339780 (lr=2.8983e-03) (hash(x)=40801937)
+1180 train 4.298719 (lr=2.8939e-03) (hash(x)=41936951)
+1190 train 4.188620 (lr=2.8893e-03) (hash(x)=41913508)
+1200 val loss 4.4159
+1200 val perplexity 82.7526
+1200 train 4.202028 (lr=2.8847e-03) (hash(x)=36842847)
+1210 train 4.298784 (lr=2.8800e-03) (hash(x)=30499991)
+1220 train 4.340014 (lr=2.8751e-03) (hash(x)=41929560)
+1230 train 4.485812 (lr=2.8702e-03) (hash(x)=47738317)
+1240 train 4.494402 (lr=2.8652e-03) (hash(x)=38256802)
+1250 train 4.481100 (lr=2.8601e-03) (hash(x)=28755106)
+1260 train 4.425727 (lr=2.8550e-03) (hash(x)=38336891)
+1270 train 4.378193 (lr=2.8497e-03) (hash(x)=36084046)
+1280 train 4.334064 (lr=2.8443e-03) (hash(x)=37921865)
+1290 train 4.417131 (lr=2.8389e-03) (hash(x)=38656933)
+1300 val loss 4.3660
+1300 val perplexity 78.7303
+1300 train 4.394099 (lr=2.8333e-03) (hash(x)=39331872)
+1310 train 4.353565 (lr=2.8277e-03) (hash(x)=35713543)
+1320 train 4.327202 (lr=2.8220e-03) (hash(x)=42221951)
+1330 train 4.306753 (lr=2.8162e-03) (hash(x)=49371177)
+1340 train 4.414113 (lr=2.8103e-03) (hash(x)=39825492)
+1350 train 4.340030 (lr=2.8044e-03) (hash(x)=38122709)
+1360 train 4.348192 (lr=2.7983e-03) (hash(x)=46034654)
+1370 train 4.082481 (lr=2.7922e-03) (hash(x)=36717950)
+1380 train 4.120973 (lr=2.7860e-03) (hash(x)=40410179)
+1390 train 4.107861 (lr=2.7797e-03) (hash(x)=46535981)
+1400 val loss 4.3467
+1400 val perplexity 77.2207
+1400 train 4.160233 (lr=2.7733e-03) (hash(x)=35556187)
+1410 train 4.265544 (lr=2.7668e-03) (hash(x)=47156509)
+1420 train 4.412834 (lr=2.7603e-03) (hash(x)=44281694)
+1430 train 4.396552 (lr=2.7536e-03) (hash(x)=38833525)
+1440 train 4.453507 (lr=2.7469e-03) (hash(x)=42715085)
+1450 train 4.492407 (lr=2.7401e-03) (hash(x)=42032181)
+1460 train 4.307829 (lr=2.7333e-03) (hash(x)=42012177)
+1470 train 4.321888 (lr=2.7263e-03) (hash(x)=41738021)
+1480 train 4.383871 (lr=2.7193e-03) (hash(x)=48868810)
+1490 train 4.347469 (lr=2.7121e-03) (hash(x)=50198350)
+1500 val loss 4.2934
+1500 val perplexity 73.2185
+1500 train 4.362706 (lr=2.7050e-03) (hash(x)=44018572)
+1510 train 4.266201 (lr=2.6977e-03) (hash(x)=40114593)
+1520 train 4.216835 (lr=2.6903e-03) (hash(x)=41475808)
+1530 train 4.290930 (lr=2.6829e-03) (hash(x)=44249710)
+1540 train 4.374002 (lr=2.6754e-03) (hash(x)=38440727)
+1550 train 4.366827 (lr=2.6678e-03) (hash(x)=38021994)
+1560 train 4.319133 (lr=2.6602e-03) (hash(x)=39234436)
+1570 train 4.217455 (lr=2.6525e-03) (hash(x)=45044408)
+1580 train 4.266973 (lr=2.6447e-03) (hash(x)=38890254)
+1590 train 4.232738 (lr=2.6368e-03) (hash(x)=39091171)
+1600 val loss 4.2598
+1600 val perplexity 70.7929
+1600 train 4.336029 (lr=2.6289e-03) (hash(x)=36528068)
+1610 train 4.221199 (lr=2.6208e-03) (hash(x)=41836863)
+1620 train 4.288796 (lr=2.6128e-03) (hash(x)=41871021)
+1630 train 4.370456 (lr=2.6046e-03) (hash(x)=41234817)
+1640 train 4.267187 (lr=2.5964e-03) (hash(x)=37912972)
+1650 train 4.193266 (lr=2.5881e-03) (hash(x)=43040972)
+1660 train 4.095904 (lr=2.5797e-03) (hash(x)=26922739)
+1670 train 4.166198 (lr=2.5713e-03) (hash(x)=39907717)
+1680 train 4.118997 (lr=2.5628e-03) (hash(x)=45004538)
+1690 train 4.105637 (lr=2.5542e-03) (hash(x)=39394756)
+1700 val loss 4.2452
+1700 val perplexity 69.7687
+1700 train 4.141019 (lr=2.5455e-03) (hash(x)=38805510)
+1710 train 4.054042 (lr=2.5368e-03) (hash(x)=34950642)
+1720 train 4.389088 (lr=2.5281e-03) (hash(x)=35167195)
+1730 train 4.385214 (lr=2.5192e-03) (hash(x)=46592240)
+1740 train 4.172699 (lr=2.5103e-03) (hash(x)=43879336)
+1750 train 4.285302 (lr=2.5014e-03) (hash(x)=42298161)
+1760 train 4.289519 (lr=2.4924e-03) (hash(x)=37358957)
+1770 train 4.234423 (lr=2.4833e-03) (hash(x)=41641994)
+1780 train 4.223020 (lr=2.4741e-03) (hash(x)=44388117)
+1790 train 4.242696 (lr=2.4649e-03) (hash(x)=45220046)
+1800 val loss 4.2041
+1800 val perplexity 66.9613
+1800 train 4.268264 (lr=2.4556e-03) (hash(x)=43748263)
+1810 train 4.077793 (lr=2.4463e-03) (hash(x)=26733445)
+1820 train 4.214490 (lr=2.4369e-03) (hash(x)=38491163)
+1830 train 4.215728 (lr=2.4275e-03) (hash(x)=35359413)
+1840 train 4.122804 (lr=2.4180e-03) (hash(x)=40482942)
+1850 train 4.013588 (lr=2.4084e-03) (hash(x)=35252918)
+1860 train 4.095462 (lr=2.3988e-03) (hash(x)=38221117)
+1870 train 4.142855 (lr=2.3891e-03) (hash(x)=41438728)
+1880 train 3.967093 (lr=2.3794e-03) (hash(x)=39801057)
+1890 train 4.166050 (lr=2.3696e-03) (hash(x)=35517965)
+1900 val loss 4.2127
+1900 val perplexity 67.5387
+1900 train 4.246020 (lr=2.3598e-03) (hash(x)=43795495)
+1910 train 4.236924 (lr=2.3499e-03) (hash(x)=47235154)
+1920 train 4.269849 (lr=2.3400e-03) (hash(x)=42139138)
+1930 train 4.215496 (lr=2.3300e-03) (hash(x)=38910301)
+1940 train 4.086425 (lr=2.3200e-03) (hash(x)=41819422)
+1950 train 4.211326 (lr=2.3099e-03) (hash(x)=44434112)
+1960 train 4.230421 (lr=2.2998e-03) (hash(x)=47468800)
+1970 train 4.282046 (lr=2.2896e-03) (hash(x)=40489494)
+1980 train 4.267266 (lr=2.2793e-03) (hash(x)=41660256)
+1990 train 4.077564 (lr=2.2691e-03) (hash(x)=36821154)
+2000 val loss 4.1867
+2000 val perplexity 65.8030
+2000 train 4.165920 (lr=2.2588e-03) (hash(x)=46230173)
+2010 train 4.140057 (lr=2.2484e-03) (hash(x)=38867399)
+2020 train 4.147109 (lr=2.2380e-03) (hash(x)=39221022)
+2030 train 4.118986 (lr=2.2275e-03) (hash(x)=40395859)
+2040 train 4.099389 (lr=2.2170e-03) (hash(x)=41260539)
+2050 train 4.094749 (lr=2.2065e-03) (hash(x)=40471948)
+2060 train 4.043067 (lr=2.1959e-03) (hash(x)=34614958)
+2070 train 4.071798 (lr=2.1853e-03) (hash(x)=40426097)
+2080 train 3.898314 (lr=2.1746e-03) (hash(x)=37618014)
+2090 train 4.204595 (lr=2.1639e-03) (hash(x)=43655438)
+2100 val loss 4.1433
+2100 val perplexity 63.0112
+2100 train 4.187691 (lr=2.1532e-03) (hash(x)=39728341)
+2110 train 4.159398 (lr=2.1424e-03) (hash(x)=40996707)
+2120 train 4.298484 (lr=2.1316e-03) (hash(x)=37072048)
+2130 train 4.114953 (lr=2.1208e-03) (hash(x)=37668129)
+2140 train 4.156023 (lr=2.1099e-03) (hash(x)=42862485)
+2150 train 4.322575 (lr=2.0990e-03) (hash(x)=40933735)
+2160 train 4.225547 (lr=2.0881e-03) (hash(x)=42952511)
+2170 train 4.098534 (lr=2.0771e-03) (hash(x)=38227732)
+2180 train 4.145338 (lr=2.0661e-03) (hash(x)=42919169)
+2190 train 4.131529 (lr=2.0550e-03) (hash(x)=38635637)
+2200 val loss 4.1393
+2200 val perplexity 62.7589
+2200 train 4.108107 (lr=2.0440e-03) (hash(x)=43974656)
+2210 train 4.095776 (lr=2.0329e-03) (hash(x)=35923510)
+2220 train 4.249701 (lr=2.0217e-03) (hash(x)=38698936)
+2230 train 3.989796 (lr=2.0106e-03) (hash(x)=41822854)
+2240 train 3.969564 (lr=1.9994e-03) (hash(x)=39069402)
+2250 train 4.022895 (lr=1.9882e-03) (hash(x)=38428746)
+2260 train 3.890714 (lr=1.9770e-03) (hash(x)=41190125)
+2270 train 4.249081 (lr=1.9657e-03) (hash(x)=39197540)
+2280 train 4.193214 (lr=1.9544e-03) (hash(x)=41964710)
+2290 train 4.057076 (lr=1.9431e-03) (hash(x)=35566499)
+2300 val loss 4.1265
+2300 val perplexity 61.9609
+2300 train 4.060265 (lr=1.9318e-03) (hash(x)=40710061)
+2310 train 4.024505 (lr=1.9205e-03) (hash(x)=34084675)
+2320 train 4.308280 (lr=1.9091e-03) (hash(x)=43192252)
+2330 train 4.207595 (lr=1.8977e-03) (hash(x)=39365674)
+2340 train 4.109126 (lr=1.8863e-03) (hash(x)=41693707)
+2350 train 4.162141 (lr=1.8749e-03) (hash(x)=41246790)
+2360 train 4.092557 (lr=1.8635e-03) (hash(x)=44379115)
+2370 train 4.051098 (lr=1.8520e-03) (hash(x)=37114071)
+2380 train 4.053510 (lr=1.8406e-03) (hash(x)=40531187)
+2390 train 4.159568 (lr=1.8291e-03) (hash(x)=35190591)
+2400 val loss 4.0920
+2400 val perplexity 59.8585
+2400 train 4.174000 (lr=1.8176e-03) (hash(x)=46483290)
+2410 train 4.015300 (lr=1.8061e-03) (hash(x)=36994364)
+2420 train 4.069708 (lr=1.7946e-03) (hash(x)=43774123)
+2430 train 4.122134 (lr=1.7830e-03) (hash(x)=39016150)
+2440 train 4.082635 (lr=1.7715e-03) (hash(x)=41424235)
+2450 train 3.906054 (lr=1.7600e-03) (hash(x)=40781424)
+2460 train 3.953524 (lr=1.7484e-03) (hash(x)=36871418)
+2470 train 3.926154 (lr=1.7368e-03) (hash(x)=40398678)
+2480 train 3.977890 (lr=1.7253e-03) (hash(x)=40019705)
+2490 train 4.025047 (lr=1.7137e-03) (hash(x)=43210658)
+2500 val loss 4.0838
+2500 val perplexity 59.3734
+2500 train 4.114532 (lr=1.7021e-03) (hash(x)=40992954)
+2510 train 4.154217 (lr=1.6906e-03) (hash(x)=40420845)
+2520 train 4.049014 (lr=1.6790e-03) (hash(x)=39278025)
+2530 train 4.033954 (lr=1.6674e-03) (hash(x)=40118798)
+2540 train 4.256076 (lr=1.6558e-03) (hash(x)=36453463)
+2550 train 4.053461 (lr=1.6442e-03) (hash(x)=38336991)
+2560 train 4.155300 (lr=1.6326e-03) (hash(x)=40307518)
+2570 train 3.963542 (lr=1.6210e-03) (hash(x)=31378688)
+2580 train 4.042953 (lr=1.6094e-03) (hash(x)=38888092)
+2590 train 4.095569 (lr=1.5979e-03) (hash(x)=43391661)
+2600 val loss 4.0566
+2600 val perplexity 57.7797
+2600 train 3.990071 (lr=1.5863e-03) (hash(x)=43784447)
+2610 train 4.043756 (lr=1.5747e-03) (hash(x)=44952165)
+2620 train 3.943068 (lr=1.5632e-03) (hash(x)=37357711)
+2630 train 4.015451 (lr=1.5516e-03) (hash(x)=41317604)
+2640 train 3.811038 (lr=1.5400e-03) (hash(x)=43473476)
+2650 train 3.964092 (lr=1.5285e-03) (hash(x)=36968500)
+2660 train 3.978471 (lr=1.5170e-03) (hash(x)=37853410)
+2670 train 3.840962 (lr=1.5054e-03) (hash(x)=55116905)
+2680 train 4.079848 (lr=1.4939e-03) (hash(x)=43789962)
+2690 train 3.959749 (lr=1.4824e-03) (hash(x)=47653410)
+2700 val loss 4.0468
+2700 val perplexity 57.2122
+2700 train 4.103662 (lr=1.4709e-03) (hash(x)=42034666)
+2710 train 4.224947 (lr=1.4594e-03) (hash(x)=34261377)
+2720 train 4.061224 (lr=1.4480e-03) (hash(x)=43776091)
+2730 train 4.094453 (lr=1.4365e-03) (hash(x)=38652143)
+2740 train 4.144126 (lr=1.4251e-03) (hash(x)=37399153)
+2750 train 4.080681 (lr=1.4137e-03) (hash(x)=41956544)
+2760 train 4.349882 (lr=1.4023e-03) (hash(x)=37661382)
+2770 train 3.994900 (lr=1.3909e-03) (hash(x)=45791682)
+2780 train 3.963238 (lr=1.3795e-03) (hash(x)=40218424)
+2790 train 3.982559 (lr=1.3682e-03) (hash(x)=38121125)
+2800 val loss 4.0294
+2800 val perplexity 56.2294
+2800 train 4.045098 (lr=1.3569e-03) (hash(x)=38892664)
+2810 train 4.023943 (lr=1.3456e-03) (hash(x)=41180129)
+2820 train 3.790727 (lr=1.3343e-03) (hash(x)=41017023)
+2830 train 3.893616 (lr=1.3230e-03) (hash(x)=42197711)
+2840 train 3.922186 (lr=1.3118e-03) (hash(x)=37993621)
+2850 train 3.830526 (lr=1.3006e-03) (hash(x)=34469615)
+2860 train 4.014174 (lr=1.2894e-03) (hash(x)=42013555)
+2870 train 4.069732 (lr=1.2783e-03) (hash(x)=37463962)
+2880 train 4.032180 (lr=1.2671e-03) (hash(x)=36874351)
+2890 train 3.996437 (lr=1.2560e-03) (hash(x)=43649626)
+2900 val loss 4.0124
+2900 val perplexity 55.2768
+2900 train 4.124608 (lr=1.2450e-03) (hash(x)=42022821)
+2910 train 3.986201 (lr=1.2339e-03) (hash(x)=36933935)
+2920 train 3.983370 (lr=1.2229e-03) (hash(x)=41313698)
+2930 train 4.023007 (lr=1.2119e-03) (hash(x)=36341574)
+2940 train 4.002404 (lr=1.2010e-03) (hash(x)=35505945)
+2950 train 4.040689 (lr=1.1901e-03) (hash(x)=47407907)
+2960 train 3.961760 (lr=1.1792e-03) (hash(x)=39217310)
+2970 train 3.973819 (lr=1.1684e-03) (hash(x)=35391019)
+2980 train 3.892808 (lr=1.1576e-03) (hash(x)=37929528)
+2990 train 4.500646 (lr=1.1468e-03) (hash(x)=42604487)
+3000 val loss 4.0054
+3000 val perplexity 54.8927
+3000 train 3.934869 (lr=1.1361e-03) (hash(x)=36439520)
+3010 train 3.787461 (lr=1.1254e-03) (hash(x)=36110173)
+3020 train 3.802625 (lr=1.1147e-03) (hash(x)=41712329)
+3030 train 3.856707 (lr=1.1041e-03) (hash(x)=39541717)
+3040 train 3.890872 (lr=1.0935e-03) (hash(x)=38203029)
+3050 train 4.218014 (lr=1.0830e-03) (hash(x)=45668569)
+3060 train 3.906355 (lr=1.0725e-03) (hash(x)=42053235)
+3070 train 3.894423 (lr=1.0620e-03) (hash(x)=38772023)
+3080 train 3.918390 (lr=1.0516e-03) (hash(x)=40842122)
+3090 train 3.866436 (lr=1.0412e-03) (hash(x)=43911820)
+3100 val loss 3.9956
+3100 val perplexity 54.3582
+3100 train 3.910733 (lr=1.0309e-03) (hash(x)=42366363)
+3110 train 3.900075 (lr=1.0207e-03) (hash(x)=42499370)
+3120 train 3.908826 (lr=1.0104e-03) (hash(x)=37964948)
+3130 train 4.023925 (lr=1.0002e-03) (hash(x)=37801102)
+3140 train 4.027692 (lr=9.9011e-04) (hash(x)=39187569)
+3150 train 4.049682 (lr=9.8002e-04) (hash(x)=37104415)
+3160 train 4.073526 (lr=9.6999e-04) (hash(x)=40414652)
+3170 train 3.764113 (lr=9.6000e-04) (hash(x)=39971875)
+3180 train 3.896595 (lr=9.5007e-04) (hash(x)=36823751)
+3190 train 3.935954 (lr=9.4019e-04) (hash(x)=41232481)
+3200 val loss 3.9770
+3200 val perplexity 53.3565
+3200 train 3.906213 (lr=9.3036e-04) (hash(x)=42263706)
+3210 train 3.932978 (lr=9.2058e-04) (hash(x)=42682932)
+3220 train 3.902016 (lr=9.1085e-04) (hash(x)=38173495)
+3230 train 3.855196 (lr=9.0118e-04) (hash(x)=42665215)
+3240 train 3.819067 (lr=8.9157e-04) (hash(x)=37430175)
+3250 train 3.878448 (lr=8.8201e-04) (hash(x)=39501818)
+3260 train 3.954237 (lr=8.7251e-04) (hash(x)=39316351)
+3270 train 4.091113 (lr=8.6307e-04) (hash(x)=33934131)
+3280 train 3.956439 (lr=8.5368e-04) (hash(x)=33684008)
+3290 train 3.958663 (lr=8.4435e-04) (hash(x)=41195688)
+3300 val loss 3.9790
+3300 val perplexity 53.4625
+3300 train 3.918606 (lr=8.3508e-04) (hash(x)=38146885)
+3310 train 3.865875 (lr=8.2588e-04) (hash(x)=41531159)
+3320 train 3.973217 (lr=8.1673e-04) (hash(x)=37465393)
+3330 train 4.001208 (lr=8.0764e-04) (hash(x)=41276314)
+3340 train 3.950613 (lr=7.9862e-04) (hash(x)=40035419)
+3350 train 3.964578 (lr=7.8966e-04) (hash(x)=38589416)
+3360 train 3.819809 (lr=7.8076e-04) (hash(x)=44616716)
+3370 train 3.939864 (lr=7.7192e-04) (hash(x)=40753545)
+3380 train 3.773901 (lr=7.6315e-04) (hash(x)=36064706)
+3390 train 3.741223 (lr=7.5445e-04) (hash(x)=35628005)
+3400 val loss 3.9593
+3400 val perplexity 52.4231
+3400 train 3.861558 (lr=7.4581e-04) (hash(x)=37242512)
+3410 train 3.916421 (lr=7.3724e-04) (hash(x)=49988219)
+3420 train 3.903453 (lr=7.2874e-04) (hash(x)=39309246)
+3430 train 4.015235 (lr=7.2030e-04) (hash(x)=41337256)
+3440 train 4.082232 (lr=7.1193e-04) (hash(x)=38422583)
+3450 train 3.869714 (lr=7.0363e-04) (hash(x)=42782162)
+3460 train 4.226733 (lr=6.9541e-04) (hash(x)=36355160)
+3470 train 3.936795 (lr=6.8725e-04) (hash(x)=43452584)
+3480 train 4.010543 (lr=6.7916e-04) (hash(x)=36009375)
+3490 train 3.879845 (lr=6.7114e-04) (hash(x)=44201141)
+3500 val loss 3.9419
+3500 val perplexity 51.5168
+3500 train 3.850629 (lr=6.6320e-04) (hash(x)=32974340)
+3510 train 3.838544 (lr=6.5533e-04) (hash(x)=38454893)
+3520 train 3.811043 (lr=6.4753e-04) (hash(x)=39406776)
+3530 train 3.861398 (lr=6.3981e-04) (hash(x)=40863480)
+3540 train 3.857097 (lr=6.3216e-04) (hash(x)=39891139)
+3550 train 4.000243 (lr=6.2458e-04) (hash(x)=44049608)
+3560 train 3.900741 (lr=6.1708e-04) (hash(x)=38049508)
+3570 train 4.006496 (lr=6.0966e-04) (hash(x)=39546422)
+3580 train 3.902916 (lr=6.0231e-04) (hash(x)=48257091)
+3590 train 3.835331 (lr=5.9504e-04) (hash(x)=36249953)
+3600 val loss 3.9306
+3600 val perplexity 50.9374
+3600 train 3.898257 (lr=5.8785e-04) (hash(x)=39324021)
+3610 train 3.876760 (lr=5.8074e-04) (hash(x)=35944408)
+3620 train 3.866680 (lr=5.7370e-04) (hash(x)=37992132)
+3630 train 3.894145 (lr=5.6675e-04) (hash(x)=37723468)
+3640 train 3.905966 (lr=5.5987e-04) (hash(x)=38300750)
+3650 train 3.926928 (lr=5.5308e-04) (hash(x)=41511149)
+3660 train 3.817177 (lr=5.4636e-04) (hash(x)=46384740)
+3670 train 3.900837 (lr=5.3973e-04) (hash(x)=40100004)
+3680 train 3.790198 (lr=5.3318e-04) (hash(x)=37889842)
+3690 train 3.908143 (lr=5.2671e-04) (hash(x)=40976637)
+3700 val loss 3.9256
+3700 val perplexity 50.6859
+3700 train 3.958560 (lr=5.2033e-04) (hash(x)=40138225)
+3710 train 3.944105 (lr=5.1402e-04) (hash(x)=38068714)
+3720 train 3.924678 (lr=5.0780e-04) (hash(x)=40819257)
+3730 train 3.876761 (lr=5.0167e-04) (hash(x)=29564095)
+3740 train 3.857092 (lr=4.9562e-04) (hash(x)=36509087)
+3750 train 4.013295 (lr=4.8965e-04) (hash(x)=41226118)
+3760 train 3.927763 (lr=4.8377e-04) (hash(x)=38317131)
+3770 train 3.924551 (lr=4.7798e-04) (hash(x)=41244909)
+3780 train 3.788678 (lr=4.7227e-04) (hash(x)=41412670)
+3790 train 3.755711 (lr=4.6665e-04) (hash(x)=34906259)
+3800 val loss 3.9183
+3800 val perplexity 50.3169
+3800 train 4.057327 (lr=4.6112e-04) (hash(x)=40968992)
+3810 train 3.773186 (lr=4.5567e-04) (hash(x)=35241519)
+3820 train 3.899414 (lr=4.5031e-04) (hash(x)=42755713)
+3830 train 3.854153 (lr=4.4504e-04) (hash(x)=47166492)
+3840 train 4.020204 (lr=4.3986e-04) (hash(x)=57200827)
+3850 train 3.883969 (lr=4.3477e-04) (hash(x)=44734592)
+3860 train 3.788623 (lr=4.2977e-04) (hash(x)=32738162)
+3870 train 3.967677 (lr=4.2486e-04) (hash(x)=39064476)
+3880 train 3.809404 (lr=4.2004e-04) (hash(x)=39921346)
+3890 train 3.933308 (lr=4.1530e-04) (hash(x)=35960386)
+3900 val loss 3.9108
+3900 val perplexity 49.9394
+3900 train 3.965215 (lr=4.1066e-04) (hash(x)=40314553)
+3910 train 3.913331 (lr=4.0611e-04) (hash(x)=43961115)
+3920 train 3.981851 (lr=4.0166e-04) (hash(x)=43423054)
+3930 train 4.169628 (lr=3.9729e-04) (hash(x)=34054777)
+3940 train 3.934535 (lr=3.9302e-04) (hash(x)=39706545)
+3950 train 3.908283 (lr=3.8884e-04) (hash(x)=37107026)
+3960 train 3.855681 (lr=3.8475e-04) (hash(x)=37669670)
+3970 train 3.931319 (lr=3.8076e-04) (hash(x)=32520030)
+3980 train 3.787637 (lr=3.7685e-04) (hash(x)=43036200)
+3990 train 3.833330 (lr=3.7305e-04) (hash(x)=39177193)
+4000 val loss 3.9016
+4000 val perplexity 49.4821
+4000 train 3.819140 (lr=3.6933e-04) (hash(x)=42698385)
+4010 train 3.772872 (lr=3.6572e-04) (hash(x)=36812123)
+4020 train 3.852390 (lr=3.6219e-04) (hash(x)=40477406)
+4030 train 3.887582 (lr=3.5876e-04) (hash(x)=43794844)
+4040 train 3.963847 (lr=3.5543e-04) (hash(x)=39871633)
+4050 train 3.905866 (lr=3.5219e-04) (hash(x)=44942955)
+4060 train 3.898297 (lr=3.4905e-04) (hash(x)=37704728)
+4070 train 4.381034 (lr=3.4600e-04) (hash(x)=44586874)
+4080 train 3.934383 (lr=3.4305e-04) (hash(x)=40698169)
+4090 train 4.026707 (lr=3.4019e-04) (hash(x)=56230743)
+4100 val loss 3.8920
+4100 val perplexity 49.0090
+4100 train 3.952113 (lr=3.3744e-04) (hash(x)=39745125)
+4110 train 3.894502 (lr=3.3477e-04) (hash(x)=38038631)
+4120 train 3.915252 (lr=3.3221e-04) (hash(x)=41225060)
+4130 train 3.943556 (lr=3.2974e-04) (hash(x)=43984411)
+4140 train 3.808182 (lr=3.2737e-04) (hash(x)=35070209)
+4150 train 3.822503 (lr=3.2510e-04) (hash(x)=44830085)
+4160 train 3.766563 (lr=3.2292e-04) (hash(x)=34909340)
+4170 train 3.967158 (lr=3.2085e-04) (hash(x)=39799332)
+4180 train 3.938090 (lr=3.1887e-04) (hash(x)=34673396)
+4190 train 3.966611 (lr=3.1699e-04) (hash(x)=43013076)
+4200 val loss 3.8850
+4200 val perplexity 48.6648
+4200 train 3.989354 (lr=3.1520e-04) (hash(x)=44540286)
+4210 train 3.944010 (lr=3.1352e-04) (hash(x)=39338462)
+4220 train 3.841686 (lr=3.1193e-04) (hash(x)=40697563)
+4230 train 3.953051 (lr=3.1044e-04) (hash(x)=36447380)
+4240 train 3.916899 (lr=3.0905e-04) (hash(x)=31542722)
+4250 train 3.818587 (lr=3.0776e-04) (hash(x)=38193385)
+4260 train 3.826108 (lr=3.0657e-04) (hash(x)=34604433)
+4270 train 3.813822 (lr=3.0548e-04) (hash(x)=34316175)
+4280 train 3.869526 (lr=3.0449e-04) (hash(x)=41834221)
+4290 train 3.872258 (lr=3.0359e-04) (hash(x)=40456686)
+4300 val loss 3.8863
+4300 val perplexity 48.7279
+4300 train 3.781909 (lr=3.0280e-04) (hash(x)=38688927)
+4310 train 3.961439 (lr=3.0210e-04) (hash(x)=41920949)
+4320 train 3.837213 (lr=3.0150e-04) (hash(x)=39588763)
+4330 train 3.877870 (lr=3.0101e-04) (hash(x)=39696346)
+4340 train 3.979786 (lr=3.0061e-04) (hash(x)=36406591)
+4350 train 3.949103 (lr=3.0031e-04) (hash(x)=34248339)
+4360 train 3.869088 (lr=3.0011e-04) (hash(x)=39031938)
+4370 train 3.898676 (lr=3.0001e-04) (hash(x)=38410485)
+4374 val loss 3.8786
+4374 val perplexity 48.3574

logs/fix_compile_bug/disable_dynamo/model_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5657bb3bb1b0e27e509e346fc962f2bb03b66978ffd11d334c52869b2cbe3b4c
+size 97706546

logs/fix_compile_bug/disable_dynamo/optimizer_04374.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8330e34f6cd87b094155ca00feb9b593ab1bf898bee64f7b1701f39bf40184f2
+size 189135414