Upload folder using huggingface_hub

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "two_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_1339", "n_embd": 256}

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2561
+0 val perplexity 77351.3281
+0 train 11.256043 (lr=1.1000e-07) (hash(x)=150724848)
+100 val loss 10.0223
+100 val perplexity 22524.0918
+100 train 10.030500 (lr=1.1110e-05) (hash(x)=149910534)
+200 val loss 9.5327
+200 val perplexity 13803.2529
+200 train 9.564865 (lr=2.2110e-05) (hash(x)=148123706)
+300 val loss 8.5776
+300 val perplexity 5311.1431
+300 train 8.496675 (lr=3.3110e-05) (hash(x)=146678221)
+400 val loss 7.9832
+400 val perplexity 2931.1692
+400 train 7.893345 (lr=4.4110e-05) (hash(x)=151700982)
+500 val loss 7.6480
+500 val perplexity 2096.4089
+500 train 7.745070 (lr=5.5000e-05) (hash(x)=156182087)
+600 val loss 7.4891
+600 val perplexity 1788.4213
+600 train 7.484365 (lr=5.4982e-05) (hash(x)=149318660)
+700 val loss 7.4051
+700 val perplexity 1644.2784
+700 train 7.444818 (lr=5.4928e-05) (hash(x)=150482428)
+800 val loss 7.3204
+800 val perplexity 1510.7963
+800 train 7.196598 (lr=5.4839e-05) (hash(x)=143268605)
+900 val loss 7.2741
+900 val perplexity 1442.4171
+900 train 7.240788 (lr=5.4713e-05) (hash(x)=152322423)
+1000 val loss 7.2295
+1000 val perplexity 1379.5698
+1000 train 7.095530 (lr=5.4553e-05) (hash(x)=147904298)
+1100 val loss 7.1887
+1100 val perplexity 1324.4363
+1100 train 7.373638 (lr=5.4357e-05) (hash(x)=154343147)
+1200 val loss 7.1326
+1200 val perplexity 1252.1884
+1200 train 7.121460 (lr=5.4126e-05) (hash(x)=141843115)
+1300 val loss 7.0963
+1300 val perplexity 1207.4440
+1300 train 6.978442 (lr=5.3860e-05) (hash(x)=145279030)
+1400 val loss 7.0511
+1400 val perplexity 1154.1615
+1400 train 7.015603 (lr=5.3561e-05) (hash(x)=152507639)
+1500 val loss 7.0244
+1500 val perplexity 1123.7080
+1500 train 7.061371 (lr=5.3227e-05) (hash(x)=148473774)
+1600 val loss 6.9760
+1600 val perplexity 1070.6720
+1600 train 7.000756 (lr=5.2860e-05) (hash(x)=151117002)
+1700 val loss 6.9307
+1700 val perplexity 1023.1615
+1700 train 6.843854 (lr=5.2461e-05) (hash(x)=138011335)
+1800 val loss 6.8845
+1800 val perplexity 977.0070
+1800 train 7.054591 (lr=5.2029e-05) (hash(x)=171180926)
+1900 val loss 6.8279
+1900 val perplexity 923.2380
+1900 train 6.733932 (lr=5.1565e-05) (hash(x)=141769419)
+2000 val loss 6.7752
+2000 val perplexity 875.8444
+2000 train 6.703566 (lr=5.1071e-05) (hash(x)=151963443)
+2100 val loss 6.7333
+2100 val perplexity 839.9230
+2100 train 6.793372 (lr=5.0547e-05) (hash(x)=162947470)
+2200 val loss 6.6757
+2200 val perplexity 792.9259
+2200 train 6.905462 (lr=4.9993e-05) (hash(x)=154954810)
+2300 val loss 6.6376
+2300 val perplexity 763.2686
+2300 train 6.630915 (lr=4.9410e-05) (hash(x)=151878111)
+2400 val loss 6.6039
+2400 val perplexity 737.9739
+2400 train 6.492835 (lr=4.8800e-05) (hash(x)=158661057)
+2500 val loss 6.5591
+2500 val perplexity 705.6418
+2500 train 6.578509 (lr=4.8162e-05) (hash(x)=150925584)
+2600 val loss 6.5238
+2600 val perplexity 681.1605
+2600 train 6.427938 (lr=4.7499e-05) (hash(x)=144515755)
+2700 val loss 6.4913
+2700 val perplexity 659.4109
+2700 train 6.476251 (lr=4.6811e-05) (hash(x)=153109144)
+2800 val loss 6.4523
+2800 val perplexity 634.1680
+2800 train 6.348439 (lr=4.6099e-05) (hash(x)=151152897)
+2900 val loss 6.4273
+2900 val perplexity 618.5321
+2900 train 6.370215 (lr=4.5364e-05) (hash(x)=145800210)
+3000 val loss 6.4106
+3000 val perplexity 608.2880
+3000 train 6.192289 (lr=4.4606e-05) (hash(x)=141997485)
+3100 val loss 6.3756
+3100 val perplexity 587.3181
+3100 train 6.274653 (lr=4.3828e-05) (hash(x)=154049740)
+3200 val loss 6.3566
+3200 val perplexity 576.2778
+3200 train 6.313905 (lr=4.3031e-05) (hash(x)=150471842)
+3300 val loss 6.3348
+3300 val perplexity 563.8449
+3300 train 6.279068 (lr=4.2215e-05) (hash(x)=149048126)
+3400 val loss 6.3060
+3400 val perplexity 547.8585
+3400 train 6.439395 (lr=4.1381e-05) (hash(x)=161261339)
+3500 val loss 6.2880
+3500 val perplexity 538.0839
+3500 train 6.264611 (lr=4.0532e-05) (hash(x)=157495564)
+3600 val loss 6.2703
+3600 val perplexity 528.6500
+3600 train 6.159868 (lr=3.9667e-05) (hash(x)=144352932)
+3700 val loss 6.2436
+3700 val perplexity 514.7292
+3700 train 6.235293 (lr=3.8789e-05) (hash(x)=149389012)
+3800 val loss 6.2293
+3800 val perplexity 507.3854
+3800 train 6.164144 (lr=3.7898e-05) (hash(x)=146607620)
+3900 val loss 6.2094
+3900 val perplexity 497.4242
+3900 train 6.122626 (lr=3.6996e-05) (hash(x)=143754617)
+4000 val loss 6.1859
+4000 val perplexity 485.8481
+4000 train 6.171858 (lr=3.6085e-05) (hash(x)=156930722)
+4100 val loss 6.1704
+4100 val perplexity 478.3918
+4100 train 5.980781 (lr=3.5165e-05) (hash(x)=147500519)
+4200 val loss 6.1567
+4200 val perplexity 471.8589
+4200 train 5.998620 (lr=3.4238e-05) (hash(x)=143232237)
+4300 val loss 6.1384
+4300 val perplexity 463.2987
+4300 train 6.063745 (lr=3.3305e-05) (hash(x)=146811670)
+4400 val loss 6.1208
+4400 val perplexity 455.2444
+4400 train 6.157179 (lr=3.2368e-05) (hash(x)=158418746)
+4500 val loss 6.1103
+4500 val perplexity 450.4887
+4500 train 6.076379 (lr=3.1428e-05) (hash(x)=156695778)
+4600 val loss 6.0926
+4600 val perplexity 442.5501
+4600 train 6.130301 (lr=3.0486e-05) (hash(x)=147791497)
+4700 val loss 6.0797
+4700 val perplexity 436.9152
+4700 train 5.948971 (lr=2.9543e-05) (hash(x)=155533088)
+4800 val loss 6.0709
+4800 val perplexity 433.0626
+4800 train 5.922207 (lr=2.8602e-05) (hash(x)=138350044)
+4900 val loss 6.0546
+4900 val perplexity 426.0846
+4900 train 6.068783 (lr=2.7663e-05) (hash(x)=143735284)
+5000 val loss 6.0431
+5000 val perplexity 421.1898
+5000 train 6.006071 (lr=2.6728e-05) (hash(x)=154976463)
+5100 val loss 6.0367
+5100 val perplexity 418.5249
+5100 train 5.913809 (lr=2.5798e-05) (hash(x)=149894982)
+5200 val loss 6.0240
+5200 val perplexity 413.2259
+5200 train 5.837811 (lr=2.4874e-05) (hash(x)=159326689)
+5300 val loss 6.0146
+5300 val perplexity 409.3807
+5300 train 6.009132 (lr=2.3958e-05) (hash(x)=159484800)
+5400 val loss 6.0013
+5400 val perplexity 403.9532
+5400 train 5.892948 (lr=2.3051e-05) (hash(x)=140385615)
+5500 val loss 5.9901
+5500 val perplexity 399.4431
+5500 train 5.821425 (lr=2.2155e-05) (hash(x)=148498335)
+5600 val loss 5.9882
+5600 val perplexity 398.7087
+5600 train 5.723786 (lr=2.1271e-05) (hash(x)=151907614)
+5700 val loss 5.9789
+5700 val perplexity 394.9890
+5700 train 5.937522 (lr=2.0399e-05) (hash(x)=155192267)
+5800 val loss 5.9621
+5800 val perplexity 388.4407
+5800 train 5.995356 (lr=1.9542e-05) (hash(x)=153132158)
+5900 val loss 5.9611
+5900 val perplexity 388.0392
+5900 train 5.921025 (lr=1.8700e-05) (hash(x)=161446764)
+6000 val loss 5.9515
+6000 val perplexity 384.3314
+6000 train 5.777972 (lr=1.7875e-05) (hash(x)=151512446)
+6100 val loss 5.9381
+6100 val perplexity 379.2043
+6100 train 6.077095 (lr=1.7068e-05) (hash(x)=188094053)
+6200 val loss 5.9274
+6200 val perplexity 375.1900
+6200 train 5.824679 (lr=1.6280e-05) (hash(x)=149389789)
+6300 val loss 5.9203
+6300 val perplexity 372.5204
+6300 train 5.733204 (lr=1.5512e-05) (hash(x)=138212820)
+6400 val loss 5.9178
+6400 val perplexity 371.5848
+6400 train 5.722038 (lr=1.4766e-05) (hash(x)=146535423)
+6500 val loss 5.9078
+6500 val perplexity 367.8837
+6500 train 5.782080 (lr=1.4042e-05) (hash(x)=145950843)
+6600 val loss 5.9012
+6600 val perplexity 365.4760
+6600 train 5.789503 (lr=1.3342e-05) (hash(x)=141162902)
+6700 val loss 5.8976
+6700 val perplexity 364.1712
+6700 train 5.800646 (lr=1.2666e-05) (hash(x)=153018737)
+6800 val loss 5.8896
+6800 val perplexity 361.2514
+6800 train 5.824938 (lr=1.2016e-05) (hash(x)=155640155)
+6900 val loss 5.8830
+6900 val perplexity 358.8943
+6900 train 5.844779 (lr=1.1392e-05) (hash(x)=153722115)
+7000 val loss 5.8770
+7000 val perplexity 356.7466
+7000 train 5.783292 (lr=1.0795e-05) (hash(x)=146953450)
+7100 val loss 5.8764
+7100 val perplexity 356.5320
+7100 train 5.695897 (lr=1.0227e-05) (hash(x)=137663885)
+7200 val loss 5.8654
+7200 val perplexity 352.6243
+7200 train 5.923510 (lr=9.6875e-06) (hash(x)=146172950)
+7300 val loss 5.8611
+7300 val perplexity 351.1057
+7300 train 5.760082 (lr=9.1780e-06) (hash(x)=150018163)
+7400 val loss 5.8582
+7400 val perplexity 350.0933
+7400 train 5.834058 (lr=8.6990e-06) (hash(x)=145351166)
+7500 val loss 5.8567
+7500 val perplexity 349.5805
+7500 train 5.641227 (lr=8.2513e-06) (hash(x)=145292116)
+7600 val loss 5.8486
+7600 val perplexity 346.7504
+7600 train 5.770745 (lr=7.8355e-06) (hash(x)=150235132)
+7700 val loss 5.8450
+7700 val perplexity 345.5066
+7700 train 5.773209 (lr=7.4522e-06) (hash(x)=154543455)
+7800 val loss 5.8428
+7800 val perplexity 344.7413
+7800 train 5.733308 (lr=7.1019e-06) (hash(x)=142456852)
+7900 val loss 5.8421
+7900 val perplexity 344.5060
+7900 train 5.617117 (lr=6.7852e-06) (hash(x)=147363479)
+8000 val loss 5.8349
+8000 val perplexity 342.0207
+8000 train 5.866343 (lr=6.5025e-06) (hash(x)=156122973)
+8100 val loss 5.8314
+8100 val perplexity 340.8327
+8100 train 5.876633 (lr=6.2543e-06) (hash(x)=156153179)
+8200 val loss 5.8302
+8200 val perplexity 340.4249
+8200 train 5.862288 (lr=6.0408e-06) (hash(x)=146430698)
+8300 val loss 5.8282
+8300 val perplexity 339.7635
+8300 train 5.688726 (lr=5.8625e-06) (hash(x)=143507257)
+8400 val loss 5.8230
+8400 val perplexity 337.9703
+8400 train 5.904530 (lr=5.7195e-06) (hash(x)=166272643)
+8500 val loss 5.8206
+8500 val perplexity 337.1700
+8500 train 5.720822 (lr=5.6121e-06) (hash(x)=143887848)
+8600 val loss 5.8196
+8600 val perplexity 336.8280
+8600 train 5.868694 (lr=5.5404e-06) (hash(x)=156900341)
+8700 val loss 5.8145
+8700 val perplexity 335.1108
+8700 train 6.027139 (lr=5.5045e-06) (hash(x)=146417632)
+8749 val loss 5.8138
+8749 val perplexity 334.9059

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36c5843c82ad1ada643ea1838b3f2dfb3a403aa559b6818e9584d59f6acf8656
+size 97580418

lr5.5e-5_total_batch_size61440_two_masks_4_heads_seed1339/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c74af2be070d22ed8df321cb00b9a59ae7f1dda9074ffb285ad3cfdba14b1eb
+size 188880262