Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads2_seed1340/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads2_seed1340/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads2_seed1340/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,33 +1,303 @@
-max_steps: 1000
+max_steps: 10000
 0 val loss 11.8210
 0 val perplexity 136085.8281
 0 train 11.833096 (lr=7.5000e-07) (hash(x)=164406924)
-100 val loss 8.9386
-100 val perplexity 7620.5469
-100 train 9.338764 (lr=7.5750e-05) (hash(x)=177407419)
-200 val loss 7.6507
-200 val perplexity 2102.1758
-200 train 7.564567 (lr=1.5000e-04) (hash(x)=144903932)
-300 val loss 7.6006
-300 val perplexity 1999.4196
-300 train 7.959331 (lr=1.4486e-04) (hash(x)=173839165)
-400 val loss 7.5781
-400 val perplexity 1954.9889
-400 train 7.667082 (lr=1.3023e-04) (hash(x)=167734596)
-500 val loss 7.5509
-500 val perplexity 1902.5231
-500 train 7.567864 (lr=1.0833e-04) (hash(x)=153224076)
-600 val loss 7.4864
-600 val perplexity 1783.6138
-600 train 7.387994 (lr=8.2500e-05) (hash(x)=149619098)
-700 val loss 7.4559
-700 val perplexity 1729.9689
-700 train 7.393428 (lr=5.6669e-05) (hash(x)=146539909)
-800 val loss 7.4234
-800 val perplexity 1674.7809
-800 train 7.390232 (lr=3.4770e-05) (hash(x)=153710890)
-900 val loss 7.4136
-900 val perplexity 1658.3834
-900 train 7.366385 (lr=2.0138e-05) (hash(x)=155873620)
-999 val loss 7.3940
-999 val perplexity 1626.1406
+100 val loss 9.2860
+100 val perplexity 10786.0391
+100 train 9.662748 (lr=7.5750e-05) (hash(x)=177407419)
+200 val loss 7.8606
+200 val perplexity 2593.0164
+200 train 7.780552 (lr=1.5000e-04) (hash(x)=144903932)
+300 val loss 7.6666
+300 val perplexity 2135.7751
+300 train 8.003074 (lr=1.4997e-04) (hash(x)=173839165)
+400 val loss 7.6502
+400 val perplexity 2101.0415
+400 train 7.739728 (lr=1.4986e-04) (hash(x)=167734596)
+500 val loss 7.6336
+500 val perplexity 2066.3887
+500 train 7.656996 (lr=1.4969e-04) (hash(x)=153224076)
+600 val loss 7.6151
+600 val perplexity 2028.6681
+600 train 7.532100 (lr=1.4945e-04) (hash(x)=149619098)
+700 val loss 7.5847
+700 val perplexity 1967.8236
+700 train 7.537936 (lr=1.4913e-04) (hash(x)=146539909)
+800 val loss 7.5700
+800 val perplexity 1939.1249
+800 train 7.540164 (lr=1.4876e-04) (hash(x)=153710890)
+900 val loss 7.5557
+900 val perplexity 1911.5392
+900 train 7.501474 (lr=1.4831e-04) (hash(x)=155873620)
+1000 val loss 7.4618
+1000 val perplexity 1740.2472
+1000 train 7.374543 (lr=1.4779e-04) (hash(x)=145450636)
+1100 val loss 7.3699
+1100 val perplexity 1587.4708
+1100 train 7.435219 (lr=1.4721e-04) (hash(x)=154123388)
+1200 val loss 7.2934
+1200 val perplexity 1470.5535
+1200 train 7.184954 (lr=1.4656e-04) (hash(x)=145249251)
+1300 val loss 7.2683
+1300 val perplexity 1434.1646
+1300 train 7.191982 (lr=1.4585e-04) (hash(x)=148937127)
+1400 val loss 7.2809
+1400 val perplexity 1452.3353
+1400 train 7.384271 (lr=1.4507e-04) (hash(x)=150475545)
+1500 val loss 7.2384
+1500 val perplexity 1391.9143
+1500 train 7.182953 (lr=1.4422e-04) (hash(x)=154653428)
+1600 val loss 7.2371
+1600 val perplexity 1390.0048
+1600 train 7.198934 (lr=1.4332e-04) (hash(x)=144483776)
+1700 val loss 7.2102
+1700 val perplexity 1353.1659
+1700 train 7.359813 (lr=1.4235e-04) (hash(x)=157395496)
+1800 val loss 7.1938
+1800 val perplexity 1331.1342
+1800 train 7.217635 (lr=1.4131e-04) (hash(x)=157916369)
+1900 val loss 7.1954
+1900 val perplexity 1333.2374
+1900 train 7.370900 (lr=1.4022e-04) (hash(x)=166073923)
+2000 val loss 7.1728
+2000 val perplexity 1303.5236
+2000 train 7.259578 (lr=1.3907e-04) (hash(x)=154856891)
+2100 val loss 7.1473
+2100 val perplexity 1270.6617
+2100 train 7.135692 (lr=1.3786e-04) (hash(x)=151925203)
+2200 val loss 7.1212
+2200 val perplexity 1237.9452
+2200 train 6.883886 (lr=1.3660e-04) (hash(x)=136191502)
+2300 val loss 7.0922
+2300 val perplexity 1202.4998
+2300 train 7.197161 (lr=1.3527e-04) (hash(x)=153273362)
+2400 val loss 7.0631
+2400 val perplexity 1168.0109
+2400 train 7.004709 (lr=1.3390e-04) (hash(x)=148021541)
+2500 val loss 7.0387
+2500 val perplexity 1139.8723
+2500 train 6.971934 (lr=1.3247e-04) (hash(x)=141356608)
+2600 val loss 7.0223
+2600 val perplexity 1121.3545
+2600 train 6.993940 (lr=1.3099e-04) (hash(x)=146005217)
+2700 val loss 6.9970
+2700 val perplexity 1093.3162
+2700 train 6.862417 (lr=1.2946e-04) (hash(x)=144511718)
+2800 val loss 6.9774
+2800 val perplexity 1072.0774
+2800 train 6.865919 (lr=1.2788e-04) (hash(x)=146019502)
+2900 val loss 6.9657
+2900 val perplexity 1059.6677
+2900 train 6.855839 (lr=1.2626e-04) (hash(x)=146496200)
+3000 val loss 6.9567
+3000 val perplexity 1050.1111
+3000 train 6.893280 (lr=1.2459e-04) (hash(x)=150127281)
+3100 val loss 6.9605
+3100 val perplexity 1054.1798
+3100 train 6.937206 (lr=1.2287e-04) (hash(x)=142022255)
+3200 val loss 6.9078
+3200 val perplexity 1000.0731
+3200 train 6.967850 (lr=1.2112e-04) (hash(x)=154120875)
+3300 val loss 6.8811
+3300 val perplexity 973.6631
+3300 train 6.926210 (lr=1.1932e-04) (hash(x)=153999717)
+3400 val loss 6.8662
+3400 val perplexity 959.2816
+3400 train 6.659736 (lr=1.1749e-04) (hash(x)=139694097)
+3500 val loss 6.8130
+3500 val perplexity 909.6061
+3500 train 7.000371 (lr=1.1562e-04) (hash(x)=162992732)
+3600 val loss 6.7788
+3600 val perplexity 879.0024
+3600 train 6.765416 (lr=1.1372e-04) (hash(x)=147574101)
+3700 val loss 6.7487
+3700 val perplexity 852.9725
+3700 train 6.844296 (lr=1.1179e-04) (hash(x)=157763099)
+3800 val loss 6.7052
+3800 val perplexity 816.6715
+3800 train 6.851088 (lr=1.0982e-04) (hash(x)=170800034)
+3900 val loss 6.6880
+3900 val perplexity 802.7108
+3900 train 6.769814 (lr=1.0783e-04) (hash(x)=164984528)
+4000 val loss 6.6738
+4000 val perplexity 791.3618
+4000 train 6.558570 (lr=1.0581e-04) (hash(x)=141743323)
+4100 val loss 6.6477
+4100 val perplexity 770.9724
+4100 train 6.665805 (lr=1.0377e-04) (hash(x)=153392872)
+4200 val loss 6.6394
+4200 val perplexity 764.6514
+4200 train 6.502887 (lr=1.0171e-04) (hash(x)=149074933)
+4300 val loss 6.6138
+4300 val perplexity 745.2818
+4300 train 6.971368 (lr=9.9622e-05) (hash(x)=167823423)
+4400 val loss 6.5934
+4400 val perplexity 730.2261
+4400 train 6.361484 (lr=9.7520e-05) (hash(x)=141203114)
+4500 val loss 6.5833
+4500 val perplexity 722.9263
+4500 train 6.580207 (lr=9.5403e-05) (hash(x)=146284780)
+4600 val loss 6.5694
+4600 val perplexity 712.9520
+4600 train 6.387619 (lr=9.3273e-05) (hash(x)=141126464)
+4700 val loss 6.5655
+4700 val perplexity 710.1322
+4700 train 6.605382 (lr=9.1132e-05) (hash(x)=154751926)
+4800 val loss 6.5409
+4800 val perplexity 692.9312
+4800 train 6.615538 (lr=8.8982e-05) (hash(x)=154793198)
+4900 val loss 6.5308
+4900 val perplexity 685.9267
+4900 train 6.277193 (lr=8.6825e-05) (hash(x)=139406392)
+5000 val loss 6.5402
+5000 val perplexity 692.4183
+5000 train 6.270849 (lr=8.4663e-05) (hash(x)=153548741)
+5100 val loss 6.5195
+5100 val perplexity 678.2264
+5100 train 6.577639 (lr=8.2500e-05) (hash(x)=160488568)
+5200 val loss 6.4913
+5200 val perplexity 659.3571
+5200 train 6.499322 (lr=8.0337e-05) (hash(x)=149645053)
+5300 val loss 6.4674
+5300 val perplexity 643.7795
+5300 train 6.551712 (lr=7.8175e-05) (hash(x)=155820556)
+5400 val loss 6.4584
+5400 val perplexity 638.0523
+5400 train 6.406846 (lr=7.6018e-05) (hash(x)=147538134)
+5500 val loss 6.4472
+5500 val perplexity 630.9539
+5500 train 6.650355 (lr=7.3868e-05) (hash(x)=166889307)
+5600 val loss 6.4414
+5600 val perplexity 627.2925
+5600 train 6.178294 (lr=7.1727e-05) (hash(x)=139516699)
+5700 val loss 6.4383
+5700 val perplexity 625.3146
+5700 train 6.140398 (lr=6.9597e-05) (hash(x)=140453511)
+5800 val loss 6.4554
+5800 val perplexity 636.1287
+5800 train 6.392017 (lr=6.7480e-05) (hash(x)=162964847)
+5900 val loss 6.4271
+5900 val perplexity 618.3743
+5900 train 6.426831 (lr=6.5378e-05) (hash(x)=150606634)
+6000 val loss 6.4019
+6000 val perplexity 602.9800
+6000 train 6.483189 (lr=6.3294e-05) (hash(x)=149890857)
+6100 val loss 6.3978
+6100 val perplexity 600.5256
+6100 train 6.496579 (lr=6.1230e-05) (hash(x)=173884145)
+6200 val loss 6.3954
+6200 val perplexity 599.0695
+6200 train 6.449534 (lr=5.9188e-05) (hash(x)=151987098)
+6300 val loss 6.3866
+6300 val perplexity 593.8149
+6300 train 6.331738 (lr=5.7169e-05) (hash(x)=148853562)
+6400 val loss 6.3854
+6400 val perplexity 593.1300
+6400 train 6.154367 (lr=5.5177e-05) (hash(x)=141530101)
+6500 val loss 6.3855
+6500 val perplexity 593.1869
+6500 train 6.248992 (lr=5.3213e-05) (hash(x)=142297809)
+6600 val loss 6.3718
+6600 val perplexity 585.1053
+6600 train 6.271256 (lr=5.1279e-05) (hash(x)=142447782)
+6700 val loss 6.3646
+6700 val perplexity 580.9097
+6700 train 6.321351 (lr=4.9377e-05) (hash(x)=147004686)
+6800 val loss 6.3596
+6800 val perplexity 578.0256
+6800 train 6.102666 (lr=4.7509e-05) (hash(x)=133438702)
+6900 val loss 6.3554
+6900 val perplexity 575.5761
+6900 train 6.356314 (lr=4.5676e-05) (hash(x)=157085143)
+7000 val loss 6.3500
+7000 val perplexity 572.5088
+7000 train 6.235214 (lr=4.3882e-05) (hash(x)=139437666)
+7100 val loss 6.3500
+7100 val perplexity 572.5020
+7100 train 6.410504 (lr=4.2128e-05) (hash(x)=159792986)
+7200 val loss 6.3324
+7200 val perplexity 562.5106
+7200 train 6.294635 (lr=4.0414e-05) (hash(x)=144930687)
+7300 val loss 6.3263
+7300 val perplexity 559.0936
+7300 train 6.391851 (lr=3.8745e-05) (hash(x)=156242690)
+7400 val loss 6.3194
+7400 val perplexity 555.2521
+7400 train 6.183175 (lr=3.7120e-05) (hash(x)=148183719)
+7500 val loss 6.3200
+7500 val perplexity 555.5490
+7500 train 6.339300 (lr=3.5541e-05) (hash(x)=152494758)
+7600 val loss 6.3193
+7600 val perplexity 555.2037
+7600 train 6.030640 (lr=3.4011e-05) (hash(x)=142485027)
+7700 val loss 6.3081
+7700 val perplexity 549.0034
+7700 train 6.164628 (lr=3.2531e-05) (hash(x)=147512165)
+7800 val loss 6.3030
+7800 val perplexity 546.2231
+7800 train 6.205514 (lr=3.1102e-05) (hash(x)=160346994)
+7900 val loss 6.3014
+7900 val perplexity 545.3183
+7900 train 6.156770 (lr=2.9726e-05) (hash(x)=144488254)
+8000 val loss 6.2974
+8000 val perplexity 543.1653
+8000 train 6.112909 (lr=2.8405e-05) (hash(x)=147637019)
+8100 val loss 6.2983
+8100 val perplexity 543.6222
+8100 train 6.162512 (lr=2.7138e-05) (hash(x)=147340534)
+8200 val loss 6.2940
+8200 val perplexity 541.3257
+8200 train 6.326890 (lr=2.5929e-05) (hash(x)=151630665)
+8300 val loss 6.2858
+8300 val perplexity 536.9080
+8300 train 6.368608 (lr=2.4778e-05) (hash(x)=149747064)
+8400 val loss 6.2828
+8400 val perplexity 535.2827
+8400 train 6.479126 (lr=2.3686e-05) (hash(x)=154245770)
+8500 val loss 6.2785
+8500 val perplexity 533.0001
+8500 train 6.166139 (lr=2.2655e-05) (hash(x)=152559100)
+8600 val loss 6.2758
+8600 val perplexity 531.5559
+8600 train 6.799290 (lr=2.1685e-05) (hash(x)=181365926)
+8700 val loss 6.2757
+8700 val perplexity 531.4955
+8700 train 6.109163 (lr=2.0777e-05) (hash(x)=154405991)
+8800 val loss 6.2743
+8800 val perplexity 530.7485
+8800 train 6.213545 (lr=1.9933e-05) (hash(x)=153755904)
+8900 val loss 6.2698
+8900 val perplexity 528.3695
+8900 train 6.213093 (lr=1.9153e-05) (hash(x)=152120568)
+9000 val loss 6.2610
+9000 val perplexity 523.7477
+9000 train 6.065022 (lr=1.8439e-05) (hash(x)=142797279)
+9100 val loss 6.2598
+9100 val perplexity 523.1348
+9100 train 6.132963 (lr=1.7790e-05) (hash(x)=143037503)
+9200 val loss 6.2580
+9200 val perplexity 522.1807
+9200 train 6.140629 (lr=1.7208e-05) (hash(x)=113690273)
+9300 val loss 6.2570
+9300 val perplexity 521.6561
+9300 train 6.183079 (lr=1.6692e-05) (hash(x)=158025077)
+9400 val loss 6.2555
+9400 val perplexity 520.8868
+9400 train 6.307523 (lr=1.6245e-05) (hash(x)=158251718)
+9500 val loss 6.2505
+9500 val perplexity 518.2725
+9500 train 6.279923 (lr=1.5865e-05) (hash(x)=154752610)
+9600 val loss 6.2469
+9600 val perplexity 516.3861
+9600 train 6.114418 (lr=1.5554e-05) (hash(x)=146889093)
+9700 val loss 6.2452
+9700 val perplexity 515.5221
+9700 train 6.245258 (lr=1.5312e-05) (hash(x)=156906516)
+9800 val loss 6.2467
+9800 val perplexity 516.2946
+9800 train 6.106436 (lr=1.5139e-05) (hash(x)=153841927)
+9900 val loss 6.2442
+9900 val perplexity 515.0154
+9900 train 6.446408 (lr=1.5035e-05) (hash(x)=163514334)
+9999 val loss 6.2399
+9999 val perplexity 512.8190

attention_kindselective_n_heads2_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35bf7922fb61d552001763039749c28ce9e91f1c804595b4ed12461463848071
+oid sha256:aa244a1ce566cb548959fd50188927b76b177114ffbf6638370c89ae1916fd2e
 size 38587970

attention_kindselective_n_heads2_seed1340/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1386979a655903e816c7a9d9e9e5fef733b0d306336f793b2b8de953ce4d1e09
+size 38587970

attention_kindselective_n_heads2_seed1340/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5becd0b3bf6aa747510ce1cbd589814a18a264f12aef236db3a20d9a0b00ff29
+size 38587970

attention_kindselective_n_heads2_seed1340/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32ff10f191a1e930111922891f17652330ad988ea68c5af5482a3fe6d08a551f
+size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:af807c8aa36ff5a8eb274063f76a52e6108fbcf1dc954f73d4b4e0fb7cd67506
+oid sha256:8b263d5796aff32bb966098ae7e921fc2175cb55e9d51196ad7423a2e310625b
 size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:236dfec65a33f5065c461861fa4cd84ca48fb9fafbd8f76b069807d997ac1ad2
+size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e059c1be6e15ded759dc20533ef62fba907eb2f6c634c3de58769a13e761a66
+size 70895430

attention_kindselective_n_heads2_seed1340/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b402bffe7717b3c7d5f831d1007da2d306e33c04f005f8a05cb1e8842ea6d10d
+size 70895430