Upload folder using huggingface_hub

attention_kindself_n_heads4_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_0/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10, "warmup_steps": 200, "group": "wider_is_better_0", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 40, "total_batch_size": 10240, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_10240_4_1340", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_4/attention_kindself_n_heads4_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1340", "n_embd": 256}

attention_kindself_n_heads4_seed1340/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindself_n_heads4_seed1340/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindself_n_heads4_seed1340/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindself_n_heads4_seed1340/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindself_n_heads4_seed1340/log2.txt

@@ -1,15 +1,303 @@
-max_steps: 10
-0 val loss 11.2844
-0 val perplexity 79566.8828
-0 train 11.278786 (lr=7.5000e-07) (hash(x)=57791809)
-1 train 11.282889 (lr=1.5000e-06) (hash(x)=51000988)
-2 train 11.271651 (lr=2.2500e-06) (hash(x)=55614127)
-3 train 11.280169 (lr=3.0000e-06) (hash(x)=50140908)
-4 train 11.264977 (lr=3.7500e-06) (hash(x)=49361285)
-5 train 11.281395 (lr=4.5000e-06) (hash(x)=43587115)
-6 train 11.254346 (lr=5.2500e-06) (hash(x)=60280800)
-7 train 11.267848 (lr=6.0000e-06) (hash(x)=55172724)
-8 train 11.241783 (lr=6.7500e-06) (hash(x)=50810700)
-9 val loss 11.2204
-9 val perplexity 74639.4609
-9 train 11.231255 (lr=7.5000e-06) (hash(x)=56912514)
+max_steps: 10000
+0 val loss 11.2821
+0 val perplexity 79387.0234
+0 train 11.281846 (lr=7.5000e-07) (hash(x)=164406924)
+100 val loss 9.3699
+100 val perplexity 11730.0283
+100 train 9.481432 (lr=7.5750e-05) (hash(x)=177407419)
+200 val loss 7.7116
+200 val perplexity 2234.0486
+200 train 7.629455 (lr=1.5000e-04) (hash(x)=144903932)
+300 val loss 7.5002
+300 val perplexity 1808.4563
+300 train 7.853703 (lr=1.4997e-04) (hash(x)=173839165)
+400 val loss 7.3591
+400 val perplexity 1570.3655
+400 train 7.457672 (lr=1.4986e-04) (hash(x)=167734596)
+500 val loss 7.2447
+500 val perplexity 1400.6857
+500 train 7.274469 (lr=1.4969e-04) (hash(x)=153224076)
+600 val loss 7.1501
+600 val perplexity 1274.2075
+600 train 7.059506 (lr=1.4945e-04) (hash(x)=149619098)
+700 val loss 7.0985
+700 val perplexity 1210.1761
+700 train 7.031834 (lr=1.4913e-04) (hash(x)=146539909)
+800 val loss 7.0450
+800 val perplexity 1147.1499
+800 train 7.017769 (lr=1.4876e-04) (hash(x)=153710890)
+900 val loss 7.0276
+900 val perplexity 1127.3376
+900 train 6.966627 (lr=1.4831e-04) (hash(x)=155873620)
+1000 val loss 7.0035
+1000 val perplexity 1100.4287
+1000 train 6.921368 (lr=1.4779e-04) (hash(x)=145450636)
+1100 val loss 6.9938
+1100 val perplexity 1089.9093
+1100 train 7.043379 (lr=1.4721e-04) (hash(x)=154123388)
+1200 val loss 6.9869
+1200 val perplexity 1082.3159
+1200 train 6.849694 (lr=1.4656e-04) (hash(x)=145249251)
+1300 val loss 6.9808
+1300 val perplexity 1075.8091
+1300 train 6.883772 (lr=1.4585e-04) (hash(x)=148937127)
+1400 val loss 6.9789
+1400 val perplexity 1073.6885
+1400 train 7.070665 (lr=1.4507e-04) (hash(x)=150475545)
+1500 val loss 6.9888
+1500 val perplexity 1084.4272
+1500 train 6.921089 (lr=1.4422e-04) (hash(x)=154653428)
+1600 val loss 7.0095
+1600 val perplexity 1107.1346
+1600 train 6.999989 (lr=1.4332e-04) (hash(x)=144483776)
+1700 val loss 6.9804
+1700 val perplexity 1075.3311
+1700 train 7.142390 (lr=1.4235e-04) (hash(x)=157395496)
+1800 val loss 6.9876
+1800 val perplexity 1083.0795
+1800 train 7.007308 (lr=1.4131e-04) (hash(x)=157916369)
+1900 val loss 6.9627
+1900 val perplexity 1056.5012
+1900 train 7.158631 (lr=1.4022e-04) (hash(x)=166073923)
+2000 val loss 6.9726
+2000 val perplexity 1067.0432
+2000 train 7.053510 (lr=1.3907e-04) (hash(x)=154856891)
+2100 val loss 6.9586
+2100 val perplexity 1052.1300
+2100 train 6.959304 (lr=1.3786e-04) (hash(x)=151925203)
+2200 val loss 6.9539
+2200 val perplexity 1047.2034
+2200 train 6.673914 (lr=1.3660e-04) (hash(x)=136191502)
+2300 val loss 6.9417
+2300 val perplexity 1034.5447
+2300 train 7.069389 (lr=1.3527e-04) (hash(x)=153273362)
+2400 val loss 6.9375
+2400 val perplexity 1030.2264
+2400 train 6.876263 (lr=1.3390e-04) (hash(x)=148021541)
+2500 val loss 6.9402
+2500 val perplexity 1033.0052
+2500 train 6.871221 (lr=1.3247e-04) (hash(x)=141356608)
+2600 val loss 6.9217
+2600 val perplexity 1014.0661
+2600 train 6.877839 (lr=1.3099e-04) (hash(x)=146005217)
+2700 val loss 6.9248
+2700 val perplexity 1017.1825
+2700 train 6.772627 (lr=1.2946e-04) (hash(x)=144511718)
+2800 val loss 6.9435
+2800 val perplexity 1036.3562
+2800 train 6.827210 (lr=1.2788e-04) (hash(x)=146019502)
+2900 val loss 6.9702
+2900 val perplexity 1064.4352
+2900 train 6.880811 (lr=1.2626e-04) (hash(x)=146496200)
+3000 val loss 6.9608
+3000 val perplexity 1054.4679
+3000 train 6.894512 (lr=1.2459e-04) (hash(x)=150127281)
+3100 val loss 6.9723
+3100 val perplexity 1066.7120
+3100 train 6.932626 (lr=1.2287e-04) (hash(x)=142022255)
+3200 val loss 6.9863
+3200 val perplexity 1081.6864
+3200 train 7.030476 (lr=1.2112e-04) (hash(x)=154120875)
+3300 val loss 6.9992
+3300 val perplexity 1095.7362
+3300 train 7.029513 (lr=1.1932e-04) (hash(x)=153999717)
+3400 val loss 6.9688
+3400 val perplexity 1062.9338
+3400 train 6.783739 (lr=1.1749e-04) (hash(x)=139694097)
+3500 val loss 7.0686
+3500 val perplexity 1174.5303
+3500 train 7.253638 (lr=1.1562e-04) (hash(x)=162992732)
+3600 val loss 7.0039
+3600 val perplexity 1100.9137
+3600 train 6.976144 (lr=1.1372e-04) (hash(x)=147574101)
+3700 val loss 6.9953
+3700 val perplexity 1091.4747
+3700 train 7.098189 (lr=1.1179e-04) (hash(x)=157763099)
+3800 val loss 7.0039
+3800 val perplexity 1100.9589
+3800 train 7.180694 (lr=1.0982e-04) (hash(x)=170800034)
+3900 val loss 7.0085
+3900 val perplexity 1106.0012
+3900 train 7.094857 (lr=1.0783e-04) (hash(x)=164984528)
+4000 val loss 7.0188
+4000 val perplexity 1117.4702
+4000 train 6.902130 (lr=1.0581e-04) (hash(x)=141743323)
+4100 val loss 7.0400
+4100 val perplexity 1141.3783
+4100 train 7.084052 (lr=1.0377e-04) (hash(x)=153392872)
+4200 val loss 7.0253
+4200 val perplexity 1124.6927
+4200 train 6.910077 (lr=1.0171e-04) (hash(x)=149074933)
+4300 val loss 7.0536
+4300 val perplexity 1157.0544
+4300 train 7.405579 (lr=9.9622e-05) (hash(x)=167823423)
+4400 val loss 7.0046
+4400 val perplexity 1101.6510
+4400 train 6.778323 (lr=9.7520e-05) (hash(x)=141203114)
+4500 val loss 6.9812
+4500 val perplexity 1076.2478
+4500 train 6.955501 (lr=9.5403e-05) (hash(x)=146284780)
+4600 val loss 6.9695
+4600 val perplexity 1063.6569
+4600 train 6.803159 (lr=9.3273e-05) (hash(x)=141126464)
+4700 val loss 6.9513
+4700 val perplexity 1044.5164
+4700 train 6.969936 (lr=9.1132e-05) (hash(x)=154751926)
+4800 val loss 6.9671
+4800 val perplexity 1061.1411
+4800 train 7.054310 (lr=8.8982e-05) (hash(x)=154793198)
+4900 val loss 6.9707
+4900 val perplexity 1064.9535
+4900 train 6.751777 (lr=8.6825e-05) (hash(x)=139406392)
+5000 val loss 6.9910
+5000 val perplexity 1086.8442
+5000 train 6.787323 (lr=8.4663e-05) (hash(x)=153548741)
+5100 val loss 6.9715
+5100 val perplexity 1065.7847
+5100 train 7.006135 (lr=8.2500e-05) (hash(x)=160488568)
+5200 val loss 6.9790
+5200 val perplexity 1073.8206
+5200 train 6.976835 (lr=8.0337e-05) (hash(x)=149645053)
+5300 val loss 6.9590
+5300 val perplexity 1052.5720
+5300 train 7.022012 (lr=7.8175e-05) (hash(x)=155820556)
+5400 val loss 6.9522
+5400 val perplexity 1045.4127
+5400 train 6.878902 (lr=7.6018e-05) (hash(x)=147538134)
+5500 val loss 6.9507
+5500 val perplexity 1043.8700
+5500 train 7.156943 (lr=7.3868e-05) (hash(x)=166889307)
+5600 val loss 6.9421
+5600 val perplexity 1034.9177
+5600 train 6.690844 (lr=7.1727e-05) (hash(x)=139516699)
+5700 val loss 6.9612
+5700 val perplexity 1054.8994
+5700 train 6.690468 (lr=6.9597e-05) (hash(x)=140453511)
+5800 val loss 6.9644
+5800 val perplexity 1058.2634
+5800 train 6.936862 (lr=6.7480e-05) (hash(x)=162964847)
+5900 val loss 6.9485
+5900 val perplexity 1041.5486
+5900 train 6.970794 (lr=6.5378e-05) (hash(x)=150606634)
+6000 val loss 6.9585
+6000 val perplexity 1052.0286
+6000 train 7.065004 (lr=6.3294e-05) (hash(x)=149890857)
+6100 val loss 6.9493
+6100 val perplexity 1042.4380
+6100 train 7.066957 (lr=6.1230e-05) (hash(x)=173884145)
+6200 val loss 6.9335
+6200 val perplexity 1026.0748
+6200 train 6.969718 (lr=5.9188e-05) (hash(x)=151987098)
+6300 val loss 6.9428
+6300 val perplexity 1035.7036
+6300 train 6.871944 (lr=5.7169e-05) (hash(x)=148853562)
+6400 val loss 6.9483
+6400 val perplexity 1041.3455
+6400 train 6.723519 (lr=5.5177e-05) (hash(x)=141530101)
+6500 val loss 6.9527
+6500 val perplexity 1045.9238
+6500 train 6.841575 (lr=5.3213e-05) (hash(x)=142297809)
+6600 val loss 6.9683
+6600 val perplexity 1062.4246
+6600 train 6.840893 (lr=5.1279e-05) (hash(x)=142447782)
+6700 val loss 6.9698
+6700 val perplexity 1064.0251
+6700 train 6.891940 (lr=4.9377e-05) (hash(x)=147004686)
+6800 val loss 6.9683
+6800 val perplexity 1062.4114
+6800 train 6.724557 (lr=4.7509e-05) (hash(x)=133438702)
+6900 val loss 6.9539
+6900 val perplexity 1047.2767
+6900 train 6.958007 (lr=4.5676e-05) (hash(x)=157085143)
+7000 val loss 6.9482
+7000 val perplexity 1041.2244
+7000 train 6.859828 (lr=4.3882e-05) (hash(x)=139437666)
+7100 val loss 6.9423
+7100 val perplexity 1035.1857
+7100 train 7.025049 (lr=4.2128e-05) (hash(x)=159792986)
+7200 val loss 6.9365
+7200 val perplexity 1029.1438
+7200 train 6.901941 (lr=4.0414e-05) (hash(x)=144930687)
+7300 val loss 6.9384
+7300 val perplexity 1031.0874
+7300 train 7.037104 (lr=3.8745e-05) (hash(x)=156242690)
+7400 val loss 6.9357
+7400 val perplexity 1028.3329
+7400 train 6.825947 (lr=3.7120e-05) (hash(x)=148183719)
+7500 val loss 6.9403
+7500 val perplexity 1033.0466
+7500 train 6.962739 (lr=3.5541e-05) (hash(x)=152494758)
+7600 val loss 6.9319
+7600 val perplexity 1024.4108
+7600 train 6.699687 (lr=3.4011e-05) (hash(x)=142485027)
+7700 val loss 6.9260
+7700 val perplexity 1018.3837
+7700 train 6.800294 (lr=3.2531e-05) (hash(x)=147512165)
+7800 val loss 6.9259
+7800 val perplexity 1018.3065
+7800 train 6.895798 (lr=3.1102e-05) (hash(x)=160346994)
+7900 val loss 6.9210
+7900 val perplexity 1013.3232
+7900 train 6.764849 (lr=2.9726e-05) (hash(x)=144488254)
+8000 val loss 6.9144
+8000 val perplexity 1006.6641
+8000 train 6.736989 (lr=2.8405e-05) (hash(x)=147637019)
+8100 val loss 6.9158
+8100 val perplexity 1008.0739
+8100 train 6.781352 (lr=2.7138e-05) (hash(x)=147340534)
+8200 val loss 6.9118
+8200 val perplexity 1004.0399
+8200 train 6.937153 (lr=2.5929e-05) (hash(x)=151630665)
+8300 val loss 6.9090
+8300 val perplexity 1001.2459
+8300 train 6.947911 (lr=2.4778e-05) (hash(x)=149747064)
+8400 val loss 6.9061
+8400 val perplexity 998.3287
+8400 train 7.081103 (lr=2.3686e-05) (hash(x)=154245770)
+8500 val loss 6.9056
+8500 val perplexity 997.8585
+8500 train 6.804367 (lr=2.2655e-05) (hash(x)=152559100)
+8600 val loss 6.9096
+8600 val perplexity 1001.8171
+8600 train 7.568511 (lr=2.1685e-05) (hash(x)=181365926)
+8700 val loss 6.9050
+8700 val perplexity 997.2687
+8700 train 6.776730 (lr=2.0777e-05) (hash(x)=154405991)
+8800 val loss 6.9002
+8800 val perplexity 992.5029
+8800 train 6.864102 (lr=1.9933e-05) (hash(x)=153755904)
+8900 val loss 6.8989
+8900 val perplexity 991.1569
+8900 train 6.848526 (lr=1.9153e-05) (hash(x)=152120568)
+9000 val loss 6.8940
+9000 val perplexity 986.3394
+9000 train 6.722706 (lr=1.8439e-05) (hash(x)=142797279)
+9100 val loss 6.8917
+9100 val perplexity 984.0582
+9100 train 6.766523 (lr=1.7790e-05) (hash(x)=143037503)
+9200 val loss 6.8935
+9200 val perplexity 985.8495
+9200 train 6.644241 (lr=1.7208e-05) (hash(x)=113690273)
+9300 val loss 6.8931
+9300 val perplexity 985.4753
+9300 train 6.832169 (lr=1.6692e-05) (hash(x)=158025077)
+9400 val loss 6.8909
+9400 val perplexity 983.2861
+9400 train 6.976372 (lr=1.6245e-05) (hash(x)=158251718)
+9500 val loss 6.8877
+9500 val perplexity 980.1492
+9500 train 6.924747 (lr=1.5865e-05) (hash(x)=154752610)
+9600 val loss 6.8858
+9600 val perplexity 978.2717
+9600 train 6.765963 (lr=1.5554e-05) (hash(x)=146889093)
+9700 val loss 6.8851
+9700 val perplexity 977.5560
+9700 train 6.876078 (lr=1.5312e-05) (hash(x)=156906516)
+9800 val loss 6.8852
+9800 val perplexity 977.6991
+9800 train 6.763077 (lr=1.5139e-05) (hash(x)=153841927)
+9900 val loss 6.8854
+9900 val perplexity 977.8618
+9900 train 7.096259 (lr=1.5035e-05) (hash(x)=163514334)
+9999 val loss 6.8815
+9999 val perplexity 974.0430

attention_kindself_n_heads4_seed1340/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86dfedb5be15ac0e3e2d448c8c6616ccbbc5669375237f4528481dc2132eb305
+oid sha256:89f7c8a60a1e0d82270696839ed2b621dad3425d4e8cff21b1af04d8c29b8563
 size 92843394

attention_kindself_n_heads4_seed1340/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d556ed16ea3cf466d90ede2b504ccb4296b4d8cae91da0286b3deb8a11d3befb
+size 92843394

attention_kindself_n_heads4_seed1340/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcf15ae166d24e9bf8468ee2bc14b4335b317df9b9a1e6aaad954dcaa96c5050
+size 92843394

attention_kindself_n_heads4_seed1340/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:661e7375a1ce344b6f39202e3f0e7d95996cbe80fc05c5b70f2abe0e870bec9d
+size 92843394

attention_kindself_n_heads4_seed1340/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19d0985ad24f878540f27a604b2c6fa0f8ed08ca0721e648c655f722775b9760
+oid sha256:13c89d6db032d55b5cd5f921e0d7f08d8b72861c8d9edfaec6437bfff2094284
 size 179406214

attention_kindself_n_heads4_seed1340/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d8dcf917931df2438081d91b4b8e3c9952f3016630c367c2ca78d80767fbab
+size 179406214

attention_kindself_n_heads4_seed1340/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9814505efbe197f3338f491dc83c65ab7fba53913755d8f98b2700318d727a05
+size 179406214

attention_kindself_n_heads4_seed1340/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2c8e4d695a273c1deb30b83a0d157100375fd74d98c01af972a5b78e6e81f846
+size 179406214