Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads2_seed1338/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads2_seed1338/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads2_seed1338/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,33 +1,303 @@
-max_steps: 1000
+max_steps: 10000
 0 val loss 11.7630
 0 val perplexity 128408.9922
 0 train 11.769258 (lr=7.5000e-07) (hash(x)=150327452)
-100 val loss 9.2739
-100 val perplexity 10656.4443
-100 train 9.258625 (lr=7.5750e-05) (hash(x)=166441190)
-200 val loss 7.7517
-200 val perplexity 2325.5278
-200 train 7.978040 (lr=1.5000e-04) (hash(x)=166780046)
-300 val loss 7.6519
-300 val perplexity 2104.7034
-300 train 7.643315 (lr=1.4486e-04) (hash(x)=159835303)
-400 val loss 7.6610
-400 val perplexity 2123.8464
-400 train 7.461915 (lr=1.3023e-04) (hash(x)=155040610)
-500 val loss 7.6780
-500 val perplexity 2160.2671
-500 train 7.390061 (lr=1.0833e-04) (hash(x)=130190460)
-600 val loss 7.6644
-600 val perplexity 2131.1912
-600 train 7.704483 (lr=8.2500e-05) (hash(x)=155504036)
-700 val loss 7.5959
-700 val perplexity 1990.0458
-700 train 7.429086 (lr=5.6669e-05) (hash(x)=137347213)
-800 val loss 7.5658
-800 val perplexity 1931.0613
-800 train 7.440493 (lr=3.4770e-05) (hash(x)=143823248)
-900 val loss 7.5432
-900 val perplexity 1887.8517
-900 train 7.664796 (lr=2.0138e-05) (hash(x)=156260416)
-999 val loss 7.5282
-999 val perplexity 1859.8395
+100 val loss 9.2668
+100 val perplexity 10581.1816
+100 train 9.254574 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.7350
+200 val perplexity 2287.0342
+200 train 7.940441 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.6488
+300 val perplexity 2098.1221
+300 train 7.643836 (lr=1.4997e-04) (hash(x)=159835303)
+400 val loss 7.6835
+400 val perplexity 2172.3113
+400 train 7.461893 (lr=1.4986e-04) (hash(x)=155040610)
+500 val loss 7.6681
+500 val perplexity 2139.0957
+500 train 7.369538 (lr=1.4969e-04) (hash(x)=130190460)
+600 val loss 7.6199
+600 val perplexity 2038.4598
+600 train 7.676965 (lr=1.4945e-04) (hash(x)=155504036)
+700 val loss 7.6129
+700 val perplexity 2024.0851
+700 train 7.439430 (lr=1.4913e-04) (hash(x)=137347213)
+800 val loss 7.5844
+800 val perplexity 1967.3245
+800 train 7.444222 (lr=1.4876e-04) (hash(x)=143823248)
+900 val loss 7.5840
+900 val perplexity 1966.5667
+900 train 7.692021 (lr=1.4831e-04) (hash(x)=156260416)
+1000 val loss 7.5333
+1000 val perplexity 1869.2717
+1000 train 7.511227 (lr=1.4779e-04) (hash(x)=143734685)
+1100 val loss 7.5359
+1100 val perplexity 1874.1538
+1100 train 7.483806 (lr=1.4721e-04) (hash(x)=160013925)
+1200 val loss 7.5170
+1200 val perplexity 1839.0571
+1200 train 7.289792 (lr=1.4656e-04) (hash(x)=150678249)
+1300 val loss 7.4718
+1300 val perplexity 1757.7659
+1300 train 7.248150 (lr=1.4585e-04) (hash(x)=149073315)
+1400 val loss 7.3732
+1400 val perplexity 1592.6691
+1400 train 7.712848 (lr=1.4507e-04) (hash(x)=175802021)
+1500 val loss 7.3131
+1500 val perplexity 1499.8285
+1500 train 7.523098 (lr=1.4422e-04) (hash(x)=171034639)
+1600 val loss 7.2376
+1600 val perplexity 1390.7593
+1600 train 7.445437 (lr=1.4332e-04) (hash(x)=158681215)
+1700 val loss 7.1747
+1700 val perplexity 1305.9194
+1700 train 7.121295 (lr=1.4235e-04) (hash(x)=152116061)
+1800 val loss 7.1227
+1800 val perplexity 1239.8262
+1800 train 7.070220 (lr=1.4131e-04) (hash(x)=146108145)
+1900 val loss 7.0877
+1900 val perplexity 1197.1328
+1900 train 6.917534 (lr=1.4022e-04) (hash(x)=147598108)
+2000 val loss 7.0835
+2000 val perplexity 1192.1318
+2000 train 6.943604 (lr=1.3907e-04) (hash(x)=154996086)
+2100 val loss 7.0599
+2100 val perplexity 1164.3746
+2100 train 6.833899 (lr=1.3786e-04) (hash(x)=153396183)
+2200 val loss 6.9674
+2200 val perplexity 1061.4762
+2200 train 6.966013 (lr=1.3660e-04) (hash(x)=153885445)
+2300 val loss 6.9418
+2300 val perplexity 1034.5950
+2300 train 6.936076 (lr=1.3527e-04) (hash(x)=159666385)
+2400 val loss 6.8840
+2400 val perplexity 976.5585
+2400 train 6.868229 (lr=1.3390e-04) (hash(x)=142353087)
+2500 val loss 6.8476
+2500 val perplexity 941.5792
+2500 train 6.853992 (lr=1.3247e-04) (hash(x)=146491718)
+2600 val loss 6.8280
+2600 val perplexity 923.3419
+2600 train 6.764425 (lr=1.3099e-04) (hash(x)=150750353)
+2700 val loss 6.8000
+2700 val perplexity 897.8646
+2700 train 6.534807 (lr=1.2946e-04) (hash(x)=129849193)
+2800 val loss 6.8085
+2800 val perplexity 905.4937
+2800 train 6.650723 (lr=1.2788e-04) (hash(x)=152767913)
+2900 val loss 6.7773
+2900 val perplexity 877.7187
+2900 train 6.550089 (lr=1.2626e-04) (hash(x)=146531140)
+3000 val loss 6.7160
+3000 val perplexity 825.5223
+3000 train 6.805177 (lr=1.2459e-04) (hash(x)=151562048)
+3100 val loss 6.6836
+3100 val perplexity 799.1540
+3100 train 6.661141 (lr=1.2287e-04) (hash(x)=146001424)
+3200 val loss 6.6572
+3200 val perplexity 778.3494
+3200 train 6.714159 (lr=1.2112e-04) (hash(x)=166486165)
+3300 val loss 6.6427
+3300 val perplexity 767.1388
+3300 train 6.582390 (lr=1.1932e-04) (hash(x)=150866680)
+3400 val loss 6.6096
+3400 val perplexity 742.1624
+3400 train 6.498703 (lr=1.1749e-04) (hash(x)=143900419)
+3500 val loss 6.6070
+3500 val perplexity 740.2560
+3500 train 6.450805 (lr=1.1562e-04) (hash(x)=148845794)
+3600 val loss 6.5769
+3600 val perplexity 718.3139
+3600 train 6.411283 (lr=1.1372e-04) (hash(x)=145667796)
+3700 val loss 6.5700
+3700 val perplexity 713.3478
+3700 train 6.569267 (lr=1.1179e-04) (hash(x)=163563851)
+3800 val loss 6.5486
+3800 val perplexity 698.2314
+3800 train 6.497487 (lr=1.0982e-04) (hash(x)=147488689)
+3900 val loss 6.4984
+3900 val perplexity 664.0471
+3900 train 6.510331 (lr=1.0783e-04) (hash(x)=148186608)
+4000 val loss 6.4814
+4000 val perplexity 652.8912
+4000 train 6.366865 (lr=1.0581e-04) (hash(x)=142970187)
+4100 val loss 6.4780
+4100 val perplexity 650.6616
+4100 train 6.527795 (lr=1.0377e-04) (hash(x)=141584883)
+4200 val loss 6.4699
+4200 val perplexity 645.4337
+4200 train 6.284593 (lr=1.0171e-04) (hash(x)=145664585)
+4300 val loss 6.4633
+4300 val perplexity 641.1999
+4300 train 6.322809 (lr=9.9622e-05) (hash(x)=143736499)
+4400 val loss 6.4581
+4400 val perplexity 637.8579
+4400 train 6.279581 (lr=9.7520e-05) (hash(x)=151883322)
+4500 val loss 6.4750
+4500 val perplexity 648.7297
+4500 train 6.346709 (lr=9.5403e-05) (hash(x)=153904871)
+4600 val loss 6.4504
+4600 val perplexity 632.9807
+4600 train 6.508712 (lr=9.3273e-05) (hash(x)=154893521)
+4700 val loss 6.4220
+4700 val perplexity 615.2382
+4700 train 6.471642 (lr=9.1132e-05) (hash(x)=152323949)
+4800 val loss 6.4123
+4800 val perplexity 609.3077
+4800 train 6.357655 (lr=8.8982e-05) (hash(x)=154104619)
+4900 val loss 6.3985
+4900 val perplexity 600.9722
+4900 train 6.501124 (lr=8.6825e-05) (hash(x)=146311426)
+5000 val loss 6.3926
+5000 val perplexity 597.3888
+5000 train 6.401734 (lr=8.4663e-05) (hash(x)=156741847)
+5100 val loss 6.3888
+5100 val perplexity 595.1534
+5100 train 6.185603 (lr=8.2500e-05) (hash(x)=142086346)
+5200 val loss 6.3876
+5200 val perplexity 594.4101
+5200 train 6.171904 (lr=8.0337e-05) (hash(x)=150265428)
+5300 val loss 6.3807
+5300 val perplexity 590.3535
+5300 train 6.253995 (lr=7.8175e-05) (hash(x)=151339108)
+5400 val loss 6.3844
+5400 val perplexity 592.5305
+5400 train 6.396504 (lr=7.6018e-05) (hash(x)=154654372)
+5500 val loss 6.3659
+5500 val perplexity 581.6820
+5500 train 6.356595 (lr=7.3868e-05) (hash(x)=150575051)
+5600 val loss 6.3722
+5600 val perplexity 585.3249
+5600 train 6.270900 (lr=7.1727e-05) (hash(x)=140396423)
+5700 val loss 6.3517
+5700 val perplexity 573.4541
+5700 train 6.227711 (lr=6.9597e-05) (hash(x)=144678758)
+5800 val loss 6.3413
+5800 val perplexity 567.5434
+5800 train 6.432989 (lr=6.7480e-05) (hash(x)=151992743)
+5900 val loss 6.3468
+5900 val perplexity 570.6630
+5900 train 6.148044 (lr=6.5378e-05) (hash(x)=144396927)
+6000 val loss 6.3284
+6000 val perplexity 560.2366
+6000 train 6.262993 (lr=6.3294e-05) (hash(x)=165478625)
+6100 val loss 6.3324
+6100 val perplexity 562.5231
+6100 train 6.103781 (lr=6.1230e-05) (hash(x)=147088621)
+6200 val loss 6.3340
+6200 val perplexity 563.4318
+6200 train 6.124668 (lr=5.9188e-05) (hash(x)=140794994)
+6300 val loss 6.3118
+6300 val perplexity 551.0154
+6300 train 6.259671 (lr=5.7169e-05) (hash(x)=134780906)
+6400 val loss 6.2949
+6400 val perplexity 541.8104
+6400 train 6.292749 (lr=5.5177e-05) (hash(x)=149023655)
+6500 val loss 6.2991
+6500 val perplexity 544.0856
+6500 train 6.181630 (lr=5.3213e-05) (hash(x)=147497796)
+6600 val loss 6.3020
+6600 val perplexity 545.6713
+6600 train 6.159567 (lr=5.1279e-05) (hash(x)=152902689)
+6700 val loss 6.2962
+6700 val perplexity 542.5115
+6700 train 6.324541 (lr=4.9377e-05) (hash(x)=153846046)
+6800 val loss 6.2718
+6800 val perplexity 529.4214
+6800 train 6.395289 (lr=4.7509e-05) (hash(x)=158512738)
+6900 val loss 6.2651
+6900 val perplexity 525.8688
+6900 train 6.748570 (lr=4.5676e-05) (hash(x)=156849968)
+7000 val loss 6.2591
+7000 val perplexity 522.7731
+7000 train 6.121258 (lr=4.3882e-05) (hash(x)=142395855)
+7100 val loss 6.2626
+7100 val perplexity 524.5578
+7100 train 6.127110 (lr=4.2128e-05) (hash(x)=147114884)
+7200 val loss 6.2535
+7200 val perplexity 519.8209
+7200 train 6.288086 (lr=4.0414e-05) (hash(x)=156979839)
+7300 val loss 6.2558
+7300 val perplexity 521.0038
+7300 train 6.026013 (lr=3.8745e-05) (hash(x)=145584373)
+7400 val loss 6.2538
+7400 val perplexity 519.9644
+7400 train 6.003152 (lr=3.7120e-05) (hash(x)=141508204)
+7500 val loss 6.2466
+7500 val perplexity 516.2325
+7500 train 6.298031 (lr=3.5541e-05) (hash(x)=148803965)
+7600 val loss 6.2317
+7600 val perplexity 508.6148
+7600 train 6.300206 (lr=3.4011e-05) (hash(x)=151019676)
+7700 val loss 6.2283
+7700 val perplexity 506.9085
+7700 train 6.276149 (lr=3.2531e-05) (hash(x)=143155750)
+7800 val loss 6.2298
+7800 val perplexity 507.6628
+7800 train 6.246758 (lr=3.1102e-05) (hash(x)=152569653)
+7900 val loss 6.2273
+7900 val perplexity 506.3650
+7900 train 6.099254 (lr=2.9726e-05) (hash(x)=143519455)
+8000 val loss 6.2230
+8000 val perplexity 504.2249
+8000 train 6.335824 (lr=2.8405e-05) (hash(x)=161180944)
+8100 val loss 6.2267
+8100 val perplexity 506.0678
+8100 train 6.229221 (lr=2.7138e-05) (hash(x)=154107345)
+8200 val loss 6.2254
+8200 val perplexity 505.4146
+8200 train 6.249242 (lr=2.5929e-05) (hash(x)=152486517)
+8300 val loss 6.2138
+8300 val perplexity 499.5728
+8300 train 6.233991 (lr=2.4778e-05) (hash(x)=156167749)
+8400 val loss 6.2082
+8400 val perplexity 496.7908
+8400 train 6.296752 (lr=2.3686e-05) (hash(x)=149155006)
+8500 val loss 6.2019
+8500 val perplexity 493.6920
+8500 train 6.154580 (lr=2.2655e-05) (hash(x)=147844390)
+8600 val loss 6.1964
+8600 val perplexity 490.9941
+8600 train 6.434650 (lr=2.1685e-05) (hash(x)=165753320)
+8700 val loss 6.1958
+8700 val perplexity 490.7081
+8700 train 5.981957 (lr=2.0777e-05) (hash(x)=146079979)
+8800 val loss 6.1972
+8800 val perplexity 491.3695
+8800 train 6.364061 (lr=1.9933e-05) (hash(x)=172259509)
+8900 val loss 6.1957
+8900 val perplexity 490.6472
+8900 train 5.909908 (lr=1.9153e-05) (hash(x)=145148314)
+9000 val loss 6.1963
+9000 val perplexity 490.9527
+9000 train 6.126988 (lr=1.8439e-05) (hash(x)=144250633)
+9100 val loss 6.1911
+9100 val perplexity 488.3842
+9100 train 6.298453 (lr=1.7790e-05) (hash(x)=157219797)
+9200 val loss 6.1824
+9200 val perplexity 484.1623
+9200 train 6.190620 (lr=1.7208e-05) (hash(x)=142743778)
+9300 val loss 6.1779
+9300 val perplexity 481.9702
+9300 train 6.096756 (lr=1.6692e-05) (hash(x)=139669771)
+9400 val loss 6.1788
+9400 val perplexity 482.4112
+9400 train 6.169431 (lr=1.6245e-05) (hash(x)=145916843)
+9500 val loss 6.1744
+9500 val perplexity 480.3114
+9500 train 6.108662 (lr=1.5865e-05) (hash(x)=150196125)
+9600 val loss 6.1753
+9600 val perplexity 480.7307
+9600 train 6.467895 (lr=1.5554e-05) (hash(x)=160041419)
+9700 val loss 6.1789
+9700 val perplexity 482.4519
+9700 train 5.973352 (lr=1.5312e-05) (hash(x)=139931627)
+9800 val loss 6.1764
+9800 val perplexity 481.2582
+9800 train 6.134768 (lr=1.5139e-05) (hash(x)=150370792)
+9900 val loss 6.1735
+9900 val perplexity 479.8646
+9900 train 6.015027 (lr=1.5035e-05) (hash(x)=153014886)
+9999 val loss 6.1745
+9999 val perplexity 480.3471

attention_kindselective_n_heads2_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34507a3447211f260cb679c78eae789754f134023cf10ab032eed11bef9db292
+oid sha256:daa37e67233d6efb44d734374df4566bbe20b2e614a268260f8db6e47dee00d3
 size 38587970

attention_kindselective_n_heads2_seed1338/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81ab9b9b3d8d5f886224514e77bbb58653393d703bda80f5d3d1b8d010267316
+size 38587970

attention_kindselective_n_heads2_seed1338/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a02fe27b98a73e6b91f79263ca68c8b309cbad8a4f154fbc3d697794471f09bc
+size 38587970

attention_kindselective_n_heads2_seed1338/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd8d934fda945273449f802b79299e9d691f001b0275cf86c919e9d7379cff50
+size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31c47e7267c8df803ebf084c7ed9ac118fe8ca35d9a86d0ba8056f3e415dcbbd
+oid sha256:6b6ea37aa2acd98cb7799f9acf8e66e04fcf30f34abb15726a52aa3aac70c926
 size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c10dfeb18a7c1d937e22d4f180ac70dc4b5c175d85eb5a5ccd7a0db3f3dd4a
+size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d9fbdb3c323d3c7c11ab626db2cde1ac3abb0a14791adc88e8b5c36f24af8b
+size 70895430

attention_kindselective_n_heads2_seed1338/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe90bda9bab22277327483ae7a8ac385591551859fbeeec7026194fc59e882d
+size 70895430