Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 100, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_4_1339", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads4_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1339", "n_embd": 256}

attention_kindselective_n_heads4_seed1339/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads4_seed1339/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads4_seed1339/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads4_seed1339/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads4_seed1339/log2.txt

@@ -1,8 +1,303 @@
-max_steps: 100
-0 val loss 11.2812
-0 val perplexity 79316.7969
-0 train 11.292645 (lr=7.5000e-07) (hash(x)=107123132)
-50 val loss 9.9065
-50 val perplexity 20060.8145
-99 val loss 8.8657
-99 val perplexity 7085.0737
+max_steps: 10000
+0 val loss 11.2810
+0 val perplexity 79298.4922
+0 train 11.295413 (lr=7.5000e-07) (hash(x)=150724848)
+100 val loss 8.6250
+100 val perplexity 5569.2051
+100 train 8.675070 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.5646
+200 val perplexity 1928.7081
+200 train 7.551589 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.4554
+300 val perplexity 1729.2274
+300 train 7.312988 (lr=1.4997e-04) (hash(x)=142185643)
+400 val loss 7.3929
+400 val perplexity 1624.3813
+400 train 7.405525 (lr=1.4986e-04) (hash(x)=148123706)
+500 val loss 7.3476
+500 val perplexity 1552.4868
+500 train 7.195226 (lr=1.4969e-04) (hash(x)=149952383)
+600 val loss 7.2760
+600 val perplexity 1445.1543
+600 train 7.072387 (lr=1.4945e-04) (hash(x)=146678221)
+700 val loss 7.2279
+700 val perplexity 1377.3153
+700 train 7.264610 (lr=1.4913e-04) (hash(x)=156180736)
+800 val loss 7.2208
+800 val perplexity 1367.6276
+800 train 7.212437 (lr=1.4876e-04) (hash(x)=151700982)
+900 val loss 7.1983
+900 val perplexity 1337.2192
+900 train 6.981043 (lr=1.4831e-04) (hash(x)=147288467)
+1000 val loss 7.2724
+1000 val perplexity 1440.0166
+1000 train 7.326461 (lr=1.4779e-04) (hash(x)=156182087)
+1100 val loss 7.2104
+1100 val perplexity 1353.4854
+1100 train 7.078388 (lr=1.4721e-04) (hash(x)=147861550)
+1200 val loss 7.2043
+1200 val perplexity 1345.2571
+1200 train 7.095018 (lr=1.4656e-04) (hash(x)=149318660)
+1300 val loss 7.1332
+1300 val perplexity 1252.8716
+1300 train 6.962538 (lr=1.4585e-04) (hash(x)=151631103)
+1400 val loss 7.0923
+1400 val perplexity 1202.7006
+1400 train 7.109515 (lr=1.4507e-04) (hash(x)=150482428)
+1500 val loss 7.0494
+1500 val perplexity 1152.1907
+1500 train 6.899590 (lr=1.4422e-04) (hash(x)=144458617)
+1600 val loss 7.0726
+1600 val perplexity 1179.2318
+1600 train 6.963625 (lr=1.4332e-04) (hash(x)=143268605)
+1700 val loss 7.0272
+1700 val perplexity 1126.8760
+1700 train 6.971726 (lr=1.4235e-04) (hash(x)=150817011)
+1800 val loss 7.0082
+1800 val perplexity 1105.6932
+1800 train 6.986873 (lr=1.4131e-04) (hash(x)=152322423)
+1900 val loss 6.9633
+1900 val perplexity 1057.0878
+1900 train 6.767703 (lr=1.4022e-04) (hash(x)=143574126)
+2000 val loss 6.9497
+2000 val perplexity 1042.8232
+2000 train 6.723602 (lr=1.3907e-04) (hash(x)=147904298)
+2100 val loss 6.9780
+2100 val perplexity 1072.7555
+2100 train 6.723686 (lr=1.3786e-04) (hash(x)=144719400)
+2200 val loss 6.9285
+2200 val perplexity 1020.9377
+2200 train 7.179186 (lr=1.3660e-04) (hash(x)=154343147)
+2300 val loss 6.8777
+2300 val perplexity 970.3969
+2300 train 6.908420 (lr=1.3527e-04) (hash(x)=154372635)
+2400 val loss 6.8547
+2400 val perplexity 948.3024
+2400 train 6.695008 (lr=1.3390e-04) (hash(x)=141843115)
+2500 val loss 6.8764
+2500 val perplexity 969.1234
+2500 train 6.780936 (lr=1.3247e-04) (hash(x)=149417679)
+2600 val loss 6.8460
+2600 val perplexity 940.0710
+2600 train 6.705399 (lr=1.3099e-04) (hash(x)=145279030)
+2700 val loss 6.8267
+2700 val perplexity 922.1456
+2700 train 6.738108 (lr=1.2946e-04) (hash(x)=146363911)
+2800 val loss 6.7896
+2800 val perplexity 888.5368
+2800 train 6.743748 (lr=1.2788e-04) (hash(x)=152507639)
+2900 val loss 6.7851
+2900 val perplexity 884.5744
+2900 train 6.561275 (lr=1.2626e-04) (hash(x)=139296191)
+3000 val loss 6.7609
+3000 val perplexity 863.4239
+3000 train 6.757187 (lr=1.2459e-04) (hash(x)=148473774)
+3100 val loss 6.6914
+3100 val perplexity 805.4599
+3100 train 6.768880 (lr=1.2287e-04) (hash(x)=154130527)
+3200 val loss 6.6348
+3200 val perplexity 761.1155
+3200 train 6.586649 (lr=1.2112e-04) (hash(x)=151117002)
+3300 val loss 6.6200
+3300 val perplexity 749.9819
+3300 train 6.764714 (lr=1.1932e-04) (hash(x)=151035383)
+3400 val loss 6.6029
+3400 val perplexity 737.2410
+3400 train 6.731507 (lr=1.1749e-04) (hash(x)=150614749)
+3500 val loss 6.5882
+3500 val perplexity 726.4773
+3500 train 6.565275 (lr=1.1562e-04) (hash(x)=151431391)
+3600 val loss 6.6085
+3600 val perplexity 741.3655
+3600 train 6.543498 (lr=1.1372e-04) (hash(x)=151140797)
+3700 val loss 6.5567
+3700 val perplexity 703.9423
+3700 train 6.750622 (lr=1.1179e-04) (hash(x)=154608340)
+3800 val loss 6.5366
+3800 val perplexity 689.9665
+3800 train 6.559993 (lr=1.0982e-04) (hash(x)=169641309)
+3900 val loss 6.5218
+3900 val perplexity 679.8000
+3900 train 6.492042 (lr=1.0783e-04) (hash(x)=150592479)
+4000 val loss 6.5130
+4000 val perplexity 673.8427
+4000 train 6.484363 (lr=1.0581e-04) (hash(x)=155495317)
+4100 val loss 6.5028
+4100 val perplexity 667.0012
+4100 train 6.692910 (lr=1.0377e-04) (hash(x)=170323970)
+4200 val loss 6.5015
+4200 val perplexity 666.1364
+4200 train 6.316307 (lr=1.0171e-04) (hash(x)=140854360)
+4300 val loss 6.4855
+4300 val perplexity 655.5953
+4300 train 6.486406 (lr=9.9622e-05) (hash(x)=148926644)
+4400 val loss 6.4532
+4400 val perplexity 634.7213
+4400 train 6.490174 (lr=9.7520e-05) (hash(x)=149997871)
+4500 val loss 6.4522
+4500 val perplexity 634.0930
+4500 train 6.414519 (lr=9.5403e-05) (hash(x)=149219289)
+4600 val loss 6.4631
+4600 val perplexity 641.0709
+4600 train 6.281563 (lr=9.3273e-05) (hash(x)=139836197)
+4700 val loss 6.4744
+4700 val perplexity 648.3419
+4700 train 6.421967 (lr=9.1132e-05) (hash(x)=150722883)
+4800 val loss 6.4518
+4800 val perplexity 633.8252
+4800 train 6.639651 (lr=8.8982e-05) (hash(x)=166864648)
+4900 val loss 6.4374
+4900 val perplexity 624.7933
+4900 train 6.549968 (lr=8.6825e-05) (hash(x)=157746743)
+5000 val loss 6.4179
+5000 val perplexity 612.7011
+5000 train 6.368554 (lr=8.4663e-05) (hash(x)=153032257)
+5100 val loss 6.4126
+5100 val perplexity 609.4542
+5100 train 6.476406 (lr=8.2500e-05) (hash(x)=151862371)
+5200 val loss 6.4037
+5200 val perplexity 604.0690
+5200 train 6.694038 (lr=8.0337e-05) (hash(x)=168862995)
+5300 val loss 6.4165
+5300 val perplexity 611.8710
+5300 train 6.378269 (lr=7.8175e-05) (hash(x)=148243567)
+5400 val loss 6.4033
+5400 val perplexity 603.8110
+5400 train 6.334872 (lr=7.6018e-05) (hash(x)=145381742)
+5500 val loss 6.3863
+5500 val perplexity 593.6646
+5500 train 6.534186 (lr=7.3868e-05) (hash(x)=155602174)
+5600 val loss 6.3683
+5600 val perplexity 583.0696
+5600 train 6.443361 (lr=7.1727e-05) (hash(x)=160637672)
+5700 val loss 6.3710
+5700 val perplexity 584.6398
+5700 train 6.291743 (lr=6.9597e-05) (hash(x)=152101131)
+5800 val loss 6.3493
+5800 val perplexity 572.1193
+5800 train 6.318473 (lr=6.7480e-05) (hash(x)=152327089)
+5900 val loss 6.3517
+5900 val perplexity 573.4702
+5900 train 6.292232 (lr=6.5378e-05) (hash(x)=151094543)
+6000 val loss 6.3507
+6000 val perplexity 572.9124
+6000 train 6.437282 (lr=6.3294e-05) (hash(x)=160233315)
+6100 val loss 6.3443
+6100 val perplexity 569.2604
+6100 train 6.436557 (lr=6.1230e-05) (hash(x)=158295337)
+6200 val loss 6.3255
+6200 val perplexity 558.6296
+6200 train 6.284213 (lr=5.9188e-05) (hash(x)=151269817)
+6300 val loss 6.3158
+6300 val perplexity 553.2232
+6300 train 6.439045 (lr=5.7169e-05) (hash(x)=156912756)
+6400 val loss 6.3036
+6400 val perplexity 546.5519
+6400 train 6.262898 (lr=5.5177e-05) (hash(x)=125779637)
+6500 val loss 6.3109
+6500 val perplexity 550.5519
+6500 train 6.200621 (lr=5.3213e-05) (hash(x)=148157661)
+6600 val loss 6.3018
+6600 val perplexity 545.5410
+6600 train 6.336145 (lr=5.1279e-05) (hash(x)=152820049)
+6700 val loss 6.2946
+6700 val perplexity 541.6356
+6700 train 6.257331 (lr=4.9377e-05) (hash(x)=155818717)
+6800 val loss 6.2828
+6800 val perplexity 535.2729
+6800 train 6.318388 (lr=4.7509e-05) (hash(x)=158833881)
+6900 val loss 6.2741
+6900 val perplexity 530.6553
+6900 train 6.197567 (lr=4.5676e-05) (hash(x)=164984909)
+7000 val loss 6.2719
+7000 val perplexity 529.5022
+7000 train 6.165509 (lr=4.3882e-05) (hash(x)=151172493)
+7100 val loss 6.2759
+7100 val perplexity 531.5972
+7100 train 6.307955 (lr=4.2128e-05) (hash(x)=154023633)
+7200 val loss 6.2661
+7200 val perplexity 526.4122
+7200 train 6.149921 (lr=4.0414e-05) (hash(x)=139351250)
+7300 val loss 6.2632
+7300 val perplexity 524.9023
+7300 train 6.218886 (lr=3.8745e-05) (hash(x)=158719458)
+7400 val loss 6.2493
+7400 val perplexity 517.6301
+7400 train 6.330258 (lr=3.7120e-05) (hash(x)=162250810)
+7500 val loss 6.2381
+7500 val perplexity 511.8631
+7500 train 6.112332 (lr=3.5541e-05) (hash(x)=142827557)
+7600 val loss 6.2380
+7600 val perplexity 511.8196
+7600 train 6.153459 (lr=3.4011e-05) (hash(x)=147830801)
+7700 val loss 6.2311
+7700 val perplexity 508.3083
+7700 train 6.335352 (lr=3.2531e-05) (hash(x)=165233314)
+7800 val loss 6.2284
+7800 val perplexity 506.9622
+7800 train 6.220976 (lr=3.1102e-05) (hash(x)=160654717)
+7900 val loss 6.2270
+7900 val perplexity 506.2233
+7900 train 6.102557 (lr=2.9726e-05) (hash(x)=140712586)
+8000 val loss 6.2113
+8000 val perplexity 498.3417
+8000 train 6.367906 (lr=2.8405e-05) (hash(x)=150309061)
+8100 val loss 6.2063
+8100 val perplexity 495.8723
+8100 train 6.201856 (lr=2.7138e-05) (hash(x)=158805466)
+8200 val loss 6.2057
+8200 val perplexity 495.5759
+8200 train 6.074079 (lr=2.5929e-05) (hash(x)=142814078)
+8300 val loss 6.1993
+8300 val perplexity 492.4088
+8300 train 6.161883 (lr=2.4778e-05) (hash(x)=148340417)
+8400 val loss 6.1987
+8400 val perplexity 492.1197
+8400 train 6.151392 (lr=2.3686e-05) (hash(x)=157132639)
+8500 val loss 6.1978
+8500 val perplexity 491.6466
+8500 train 6.153172 (lr=2.2655e-05) (hash(x)=152994240)
+8600 val loss 6.1860
+8600 val perplexity 485.8925
+8600 train 6.145603 (lr=2.1685e-05) (hash(x)=152210154)
+8700 val loss 6.1859
+8700 val perplexity 485.8365
+8700 train 6.241817 (lr=2.0777e-05) (hash(x)=149631247)
+8800 val loss 6.1828
+8800 val perplexity 484.3528
+8800 train 6.051007 (lr=1.9933e-05) (hash(x)=145617784)
+8900 val loss 6.1769
+8900 val perplexity 481.4896
+8900 train 6.254554 (lr=1.9153e-05) (hash(x)=155984970)
+9000 val loss 6.1798
+9000 val perplexity 482.8825
+9000 train 5.940945 (lr=1.8439e-05) (hash(x)=143433013)
+9100 val loss 6.1810
+9100 val perplexity 483.4917
+9100 train 6.057093 (lr=1.7790e-05) (hash(x)=152507533)
+9200 val loss 6.1740
+9200 val perplexity 480.1161
+9200 train 6.288232 (lr=1.7208e-05) (hash(x)=162400650)
+9300 val loss 6.1685
+9300 val perplexity 477.4570
+9300 train 6.225449 (lr=1.6692e-05) (hash(x)=148419068)
+9400 val loss 6.1678
+9400 val perplexity 477.1325
+9400 train 6.139578 (lr=1.6245e-05) (hash(x)=145025169)
+9500 val loss 6.1637
+9500 val perplexity 475.2030
+9500 train 6.133398 (lr=1.5865e-05) (hash(x)=145104235)
+9600 val loss 6.1667
+9600 val perplexity 476.6320
+9600 train 6.100382 (lr=1.5554e-05) (hash(x)=150499797)
+9700 val loss 6.1654
+9700 val perplexity 476.0076
+9700 train 5.987143 (lr=1.5312e-05) (hash(x)=139690022)
+9800 val loss 6.1580
+9800 val perplexity 472.4623
+9800 train 6.160158 (lr=1.5139e-05) (hash(x)=145645279)
+9900 val loss 6.1531
+9900 val perplexity 470.1617
+9900 train 6.052269 (lr=1.5035e-05) (hash(x)=150103432)
+9999 val loss 6.1522
+9999 val perplexity 469.7554

attention_kindselective_n_heads4_seed1339/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f29d95da43ad1c934a676465bbc862a3ce48bdf447f56a3ef009f429d0e0b535
+oid sha256:e716d019135dfe3a2162e5557ea36fbb3de4f3a202bd9edbebb7262813fb5f5f
 size 92843394

attention_kindselective_n_heads4_seed1339/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a29171b6ea1a71ea214bfb3b23186077674ba2b36ea6818a1ecb25d083c5a24
+size 92843394

attention_kindselective_n_heads4_seed1339/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cc9582bdfd4241e8a7e5a7ac2d18b080ce3fc67ed7721340f67a55d44f36b1c
+size 92843394

attention_kindselective_n_heads4_seed1339/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c20142fb9b1c16799d1e6996ace274d6582244a691ec9343774e0f59016276ab
+size 92843394

attention_kindselective_n_heads4_seed1339/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27c1c05dfbdb6446c9bc405288fa223e02d457671ad093ed57ceae8abc58059b
+oid sha256:bb13a645e3fb0d0a7ad40ae6b27063e7881bc03ee558ced863ad4d5e2b4e0c43
 size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56b9061bc444d656c038eb3f035450cd31039dd78d555ec8510c390924b9e5ac
+size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537491d8da9df92ce0fe0f70ddbf61245ac4a12374ab5b94c15fd36c90057ff5
+size 179406214

attention_kindselective_n_heads4_seed1339/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:514e3f12687baea816c1ef563b9a785864000f1cb4641b8ced72d92cdab0c005
+size 179406214