Upload folder using huggingface_hub

attention_kindselective_n_heads4_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 100, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_4_1338", "n_embd": 256}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_3/attention_kindselective_n_heads4_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 10000, "warmup_steps": 200, "group": "wider_is_better_3", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_4_1338", "n_embd": 256}

attention_kindselective_n_heads4_seed1338/dataloader_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8b2ea67f78ff5a7970d0db044ff7ee527b3dc065f295fd30f588df4b44b568d0
+oid sha256:db41c5e5513137877487a93451adf8ec4ed2448ab6e9471ebd5595c8e3293875
 size 964

attention_kindselective_n_heads4_seed1338/dataloader_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6df8947c6ab773db1947914387d3db345a84828521d3a64bae9b652e1b0a410
+size 964

attention_kindselective_n_heads4_seed1338/dataloader_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169891a726a7ff746d1a7aa99f459a66d85ceb4e9f2583f790f5b8501f97b6af
+size 964

attention_kindselective_n_heads4_seed1338/dataloader_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e685a568a36c792ccbe7b5fcae0b9d630955e589991190bd8902836cea6a91df
+size 964

attention_kindselective_n_heads4_seed1338/log2.txt

@@ -1,8 +1,303 @@
-max_steps: 100
-0 val loss 11.2679
-0 val perplexity 78266.4062
-0 train 11.265170 (lr=7.5000e-07) (hash(x)=99006591)
-50 val loss 9.8703
-50 val perplexity 19346.5938
-99 val loss 8.9645
-99 val perplexity 7820.4814
+max_steps: 10000
+0 val loss 11.2663
+0 val perplexity 78147.0000
+0 train 11.263341 (lr=7.5000e-07) (hash(x)=150327452)
+100 val loss 8.6550
+100 val perplexity 5739.0356
+100 train 8.665512 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.5622
+200 val perplexity 1924.0106
+200 train 7.762118 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.4673
+300 val perplexity 1749.8013
+300 train 7.422103 (lr=1.4997e-04) (hash(x)=159835303)
+400 val loss 7.4181
+400 val perplexity 1665.9224
+400 train 7.179411 (lr=1.4986e-04) (hash(x)=155040610)
+500 val loss 7.3887
+500 val perplexity 1617.5394
+500 train 7.086969 (lr=1.4969e-04) (hash(x)=130190460)
+600 val loss 7.3440
+600 val perplexity 1546.8597
+600 train 7.368630 (lr=1.4945e-04) (hash(x)=155504036)
+700 val loss 7.3203
+700 val perplexity 1510.6406
+700 train 7.165936 (lr=1.4913e-04) (hash(x)=137347213)
+800 val loss 7.2788
+800 val perplexity 1449.2776
+800 train 7.137177 (lr=1.4876e-04) (hash(x)=143823248)
+900 val loss 7.2363
+900 val perplexity 1388.9182
+900 train 7.323705 (lr=1.4831e-04) (hash(x)=156260416)
+1000 val loss 7.2580
+1000 val perplexity 1419.4784
+1000 train 7.215821 (lr=1.4779e-04) (hash(x)=143734685)
+1100 val loss 7.3114
+1100 val perplexity 1497.2897
+1100 train 7.257675 (lr=1.4721e-04) (hash(x)=160013925)
+1200 val loss 7.3346
+1200 val perplexity 1532.4198
+1200 train 7.167082 (lr=1.4656e-04) (hash(x)=150678249)
+1300 val loss 7.2587
+1300 val perplexity 1420.4479
+1300 train 7.033679 (lr=1.4585e-04) (hash(x)=149073315)
+1400 val loss 7.2140
+1400 val perplexity 1358.2969
+1400 train 7.497233 (lr=1.4507e-04) (hash(x)=175802021)
+1500 val loss 7.1960
+1500 val perplexity 1334.0533
+1500 train 7.436237 (lr=1.4422e-04) (hash(x)=171034639)
+1600 val loss 7.1795
+1600 val perplexity 1312.1996
+1600 train 7.375320 (lr=1.4332e-04) (hash(x)=158681215)
+1700 val loss 7.1637
+1700 val perplexity 1291.6211
+1700 train 7.105666 (lr=1.4235e-04) (hash(x)=152116061)
+1800 val loss 7.1537
+1800 val perplexity 1278.7939
+1800 train 7.099223 (lr=1.4131e-04) (hash(x)=146108145)
+1900 val loss 7.1225
+1900 val perplexity 1239.5602
+1900 train 6.953999 (lr=1.4022e-04) (hash(x)=147598108)
+2000 val loss 7.1305
+2000 val perplexity 1249.5562
+2000 train 6.993223 (lr=1.3907e-04) (hash(x)=154996086)
+2100 val loss 7.1265
+2100 val perplexity 1244.4935
+2100 train 6.919074 (lr=1.3786e-04) (hash(x)=153396183)
+2200 val loss 7.0872
+2200 val perplexity 1196.5553
+2200 train 7.082876 (lr=1.3660e-04) (hash(x)=153885445)
+2300 val loss 7.0497
+2300 val perplexity 1152.4731
+2300 train 7.037508 (lr=1.3527e-04) (hash(x)=159666385)
+2400 val loss 7.0609
+2400 val perplexity 1165.5144
+2400 train 7.040409 (lr=1.3390e-04) (hash(x)=142353087)
+2500 val loss 7.0770
+2500 val perplexity 1184.3530
+2500 train 7.086487 (lr=1.3247e-04) (hash(x)=146491718)
+2600 val loss 7.0654
+2600 val perplexity 1170.7280
+2600 train 7.002256 (lr=1.3099e-04) (hash(x)=150750353)
+2700 val loss 7.0348
+2700 val perplexity 1135.4803
+2700 train 6.780224 (lr=1.2946e-04) (hash(x)=129849193)
+2800 val loss 7.0484
+2800 val perplexity 1151.0122
+2800 train 6.927810 (lr=1.2788e-04) (hash(x)=152767913)
+2900 val loss 7.0414
+2900 val perplexity 1143.0150
+2900 train 6.818239 (lr=1.2626e-04) (hash(x)=146531140)
+3000 val loss 7.0160
+3000 val perplexity 1114.3568
+3000 train 7.076974 (lr=1.2459e-04) (hash(x)=151562048)
+3100 val loss 6.9971
+3100 val perplexity 1093.4324
+3100 train 6.987829 (lr=1.2287e-04) (hash(x)=146001424)
+3200 val loss 6.9538
+3200 val perplexity 1047.0964
+3200 train 7.006474 (lr=1.2112e-04) (hash(x)=166486165)
+3300 val loss 6.9583
+3300 val perplexity 1051.8140
+3300 train 6.897943 (lr=1.1932e-04) (hash(x)=150866680)
+3400 val loss 6.9559
+3400 val perplexity 1049.2837
+3400 train 6.872110 (lr=1.1749e-04) (hash(x)=143900419)
+3500 val loss 6.9829
+3500 val perplexity 1078.0233
+3500 train 6.830639 (lr=1.1562e-04) (hash(x)=148845794)
+3600 val loss 6.9487
+3600 val perplexity 1041.7463
+3600 train 6.816872 (lr=1.1372e-04) (hash(x)=145667796)
+3700 val loss 6.9561
+3700 val perplexity 1049.5679
+3700 train 6.973476 (lr=1.1179e-04) (hash(x)=163563851)
+3800 val loss 6.9661
+3800 val perplexity 1060.0791
+3800 train 6.935971 (lr=1.0982e-04) (hash(x)=147488689)
+3900 val loss 6.9352
+3900 val perplexity 1027.8176
+3900 train 6.920702 (lr=1.0783e-04) (hash(x)=148186608)
+4000 val loss 6.8950
+4000 val perplexity 987.3676
+4000 train 6.763925 (lr=1.0581e-04) (hash(x)=142970187)
+4100 val loss 6.9069
+4100 val perplexity 999.1064
+4100 train 6.910237 (lr=1.0377e-04) (hash(x)=141584883)
+4200 val loss 6.8708
+4200 val perplexity 963.6940
+4200 train 6.701746 (lr=1.0171e-04) (hash(x)=145664585)
+4300 val loss 6.8682
+4300 val perplexity 961.1713
+4300 train 6.726182 (lr=9.9622e-05) (hash(x)=143736499)
+4400 val loss 6.8514
+4400 val perplexity 945.1613
+4400 train 6.674716 (lr=9.7520e-05) (hash(x)=151883322)
+4500 val loss 6.8442
+4500 val perplexity 938.4389
+4500 train 6.743720 (lr=9.5403e-05) (hash(x)=153904871)
+4600 val loss 6.8544
+4600 val perplexity 947.9967
+4600 train 6.910264 (lr=9.3273e-05) (hash(x)=154893521)
+4700 val loss 6.8130
+4700 val perplexity 909.5671
+4700 train 6.868094 (lr=9.1132e-05) (hash(x)=152323949)
+4800 val loss 6.7850
+4800 val perplexity 884.5167
+4800 train 6.726544 (lr=8.8982e-05) (hash(x)=154104619)
+4900 val loss 6.7750
+4900 val perplexity 875.6627
+4900 train 6.839969 (lr=8.6825e-05) (hash(x)=146311426)
+5000 val loss 6.7802
+5000 val perplexity 880.2397
+5000 train 6.811939 (lr=8.4663e-05) (hash(x)=156741847)
+5100 val loss 6.8186
+5100 val perplexity 914.6631
+5100 train 6.625038 (lr=8.2500e-05) (hash(x)=142086346)
+5200 val loss 6.8095
+5200 val perplexity 906.4241
+5200 train 6.609668 (lr=8.0337e-05) (hash(x)=150265428)
+5300 val loss 6.8070
+5300 val perplexity 904.1855
+5300 train 6.721405 (lr=7.8175e-05) (hash(x)=151339108)
+5400 val loss 6.7747
+5400 val perplexity 875.4039
+5400 train 6.778082 (lr=7.6018e-05) (hash(x)=154654372)
+5500 val loss 6.7680
+5500 val perplexity 869.5363
+5500 train 6.757663 (lr=7.3868e-05) (hash(x)=150575051)
+5600 val loss 6.7493
+5600 val perplexity 853.4575
+5600 train 6.652130 (lr=7.1727e-05) (hash(x)=140396423)
+5700 val loss 6.7432
+5700 val perplexity 848.2428
+5700 train 6.634809 (lr=6.9597e-05) (hash(x)=144678758)
+5800 val loss 6.7379
+5800 val perplexity 843.7819
+5800 train 6.839241 (lr=6.7480e-05) (hash(x)=151992743)
+5900 val loss 6.7451
+5900 val perplexity 849.8788
+5900 train 6.541656 (lr=6.5378e-05) (hash(x)=144396927)
+6000 val loss 6.7338
+6000 val perplexity 840.3497
+6000 train 6.688858 (lr=6.3294e-05) (hash(x)=165478625)
+6100 val loss 6.7536
+6100 val perplexity 857.1795
+6100 train 6.538942 (lr=6.1230e-05) (hash(x)=147088621)
+6200 val loss 6.7280
+6200 val perplexity 835.4413
+6200 train 6.533285 (lr=5.9188e-05) (hash(x)=140794994)
+6300 val loss 6.7068
+6300 val perplexity 817.9677
+6300 train 6.634907 (lr=5.7169e-05) (hash(x)=134780906)
+6400 val loss 6.6841
+6400 val perplexity 799.6194
+6400 train 6.686557 (lr=5.5177e-05) (hash(x)=149023655)
+6500 val loss 6.6806
+6500 val perplexity 796.8002
+6500 train 6.581760 (lr=5.3213e-05) (hash(x)=147497796)
+6600 val loss 6.6870
+6600 val perplexity 801.9154
+6600 train 6.556812 (lr=5.1279e-05) (hash(x)=152902689)
+6700 val loss 6.6708
+6700 val perplexity 789.0045
+6700 train 6.713040 (lr=4.9377e-05) (hash(x)=153846046)
+6800 val loss 6.6602
+6800 val perplexity 780.6896
+6800 train 6.773557 (lr=4.7509e-05) (hash(x)=158512738)
+6900 val loss 6.6533
+6900 val perplexity 775.3393
+6900 train 7.021071 (lr=4.5676e-05) (hash(x)=156849968)
+7000 val loss 6.6413
+7000 val perplexity 766.1014
+7000 train 6.475348 (lr=4.3882e-05) (hash(x)=142395855)
+7100 val loss 6.6401
+7100 val perplexity 765.1912
+7100 train 6.519809 (lr=4.2128e-05) (hash(x)=147114884)
+7200 val loss 6.6253
+7200 val perplexity 753.9130
+7200 train 6.653028 (lr=4.0414e-05) (hash(x)=156979839)
+7300 val loss 6.6312
+7300 val perplexity 758.3539
+7300 train 6.406144 (lr=3.8745e-05) (hash(x)=145584373)
+7400 val loss 6.6381
+7400 val perplexity 763.6647
+7400 train 6.395841 (lr=3.7120e-05) (hash(x)=141508204)
+7500 val loss 6.6150
+7500 val perplexity 746.1946
+7500 train 6.663067 (lr=3.5541e-05) (hash(x)=148803965)
+7600 val loss 6.6029
+7600 val perplexity 737.2593
+7600 train 6.661316 (lr=3.4011e-05) (hash(x)=151019676)
+7700 val loss 6.6007
+7700 val perplexity 735.5807
+7700 train 6.594604 (lr=3.2531e-05) (hash(x)=143155750)
+7800 val loss 6.5994
+7800 val perplexity 734.6207
+7800 train 6.602177 (lr=3.1102e-05) (hash(x)=152569653)
+7900 val loss 6.5959
+7900 val perplexity 732.0568
+7900 train 6.470634 (lr=2.9726e-05) (hash(x)=143519455)
+8000 val loss 6.5947
+8000 val perplexity 731.1951
+8000 train 6.702043 (lr=2.8405e-05) (hash(x)=161180944)
+8100 val loss 6.5933
+8100 val perplexity 730.1704
+8100 train 6.609785 (lr=2.7138e-05) (hash(x)=154107345)
+8200 val loss 6.5931
+8200 val perplexity 730.0322
+8200 train 6.603320 (lr=2.5929e-05) (hash(x)=152486517)
+8300 val loss 6.5839
+8300 val perplexity 723.3425
+8300 train 6.612710 (lr=2.4778e-05) (hash(x)=156167749)
+8400 val loss 6.5769
+8400 val perplexity 718.2834
+8400 train 6.650872 (lr=2.3686e-05) (hash(x)=149155006)
+8500 val loss 6.5718
+8500 val perplexity 714.6675
+8500 train 6.545753 (lr=2.2655e-05) (hash(x)=147844390)
+8600 val loss 6.5630
+8600 val perplexity 708.4192
+8600 train 6.785943 (lr=2.1685e-05) (hash(x)=165753320)
+8700 val loss 6.5601
+8700 val perplexity 706.3683
+8700 train 6.360015 (lr=2.0777e-05) (hash(x)=146079979)
+8800 val loss 6.5567
+8800 val perplexity 703.9554
+8800 train 6.748188 (lr=1.9933e-05) (hash(x)=172259509)
+8900 val loss 6.5590
+8900 val perplexity 705.5823
+8900 train 6.276247 (lr=1.9153e-05) (hash(x)=145148314)
+9000 val loss 6.5564
+9000 val perplexity 703.7161
+9000 train 6.482366 (lr=1.8439e-05) (hash(x)=144250633)
+9100 val loss 6.5477
+9100 val perplexity 697.6514
+9100 train 6.653626 (lr=1.7790e-05) (hash(x)=157219797)
+9200 val loss 6.5406
+9200 val perplexity 692.6686
+9200 train 6.510061 (lr=1.7208e-05) (hash(x)=142743778)
+9300 val loss 6.5375
+9300 val perplexity 690.5378
+9300 train 6.438917 (lr=1.6692e-05) (hash(x)=139669771)
+9400 val loss 6.5340
+9400 val perplexity 688.1193
+9400 train 6.500862 (lr=1.6245e-05) (hash(x)=145916843)
+9500 val loss 6.5310
+9500 val perplexity 686.0958
+9500 train 6.452840 (lr=1.5865e-05) (hash(x)=150196125)
+9600 val loss 6.5286
+9600 val perplexity 684.4724
+9600 train 6.808760 (lr=1.5554e-05) (hash(x)=160041419)
+9700 val loss 6.5288
+9700 val perplexity 684.5671
+9700 train 6.348433 (lr=1.5312e-05) (hash(x)=139931627)
+9800 val loss 6.5260
+9800 val perplexity 682.6550
+9800 train 6.475728 (lr=1.5139e-05) (hash(x)=150370792)
+9900 val loss 6.5226
+9900 val perplexity 680.3587
+9900 train 6.393685 (lr=1.5035e-05) (hash(x)=153014886)
+9999 val loss 6.5211
+9999 val perplexity 679.3424

attention_kindselective_n_heads4_seed1338/model_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0910a3ff989b4f9945600f7b5cf7fc796b709bf3d075f99ae9f9c56b7acc5adb
+oid sha256:d791b9df5a73ce7373c8c3633d7ef61a92efffb082c40753771a5af9686616b6
 size 92843394

attention_kindselective_n_heads4_seed1338/model_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec252cc9736a9564ca93925735d67637349118511176e491ebd99ebc5ab280e2
+size 92843394

attention_kindselective_n_heads4_seed1338/model_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2c3687223cc276736023709fb477bf883ed750f52a3733196fbe90fb0c3d493
+size 92843394

attention_kindselective_n_heads4_seed1338/model_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d22cc128cf4ed809ae94894629d9b6cb11500983fdc2887db59483fc59d84406
+size 92843394

attention_kindselective_n_heads4_seed1338/optimizer_02500.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad38d096803391d0fada528297bf1e72e52f396b81bf5b18cfdd36287f6f4c77
+oid sha256:4957bc0a1124b62de2ad847f9bab34b5be4f4bf18c8d7e6e908770af55cbbff8
 size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_05000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5ec9251ee7644d9c14036dded38a955e9e61897405fc03587dff8d4614fa33b
+size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_07500.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71e112b5d7a75337aac523f9377464968f888fd2fb95b1c001e11144064dad91
+size 179406214

attention_kindselective_n_heads4_seed1338/optimizer_09999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7af355bdcbdda09d35c074c218779f17c5de3a922e738ba2fb1ee7c2802edb5
+size 179406214