Upload folder using huggingface_hub

lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 5.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "5.5e-5_61440_1340", "n_embd": 256}

lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2917
+0 val perplexity 80150.6406
+0 train 11.290673 (lr=1.1000e-07) (hash(x)=164406924)
+100 val loss 10.1618
+100 val perplexity 25894.9746
+100 train 10.113302 (lr=1.1110e-05) (hash(x)=144903932)
+200 val loss 9.8351
+200 val perplexity 18677.0645
+200 train 9.883295 (lr=2.2110e-05) (hash(x)=167734596)
+300 val loss 9.6368
+300 val perplexity 15318.5947
+300 train 9.672400 (lr=3.3110e-05) (hash(x)=149619098)
+400 val loss 9.3037
+400 val perplexity 10978.2139
+400 train 9.291905 (lr=4.4110e-05) (hash(x)=153710890)
+500 val loss 9.0906
+500 val perplexity 8871.8887
+500 train 9.091104 (lr=5.5000e-05) (hash(x)=145450636)
+600 val loss 9.0004
+600 val perplexity 8106.2065
+600 train 8.912514 (lr=5.4982e-05) (hash(x)=145249251)
+700 val loss 8.5919
+700 val perplexity 5387.6914
+700 train 8.676911 (lr=5.4928e-05) (hash(x)=150475545)
+800 val loss 8.0865
+800 val perplexity 3250.3789
+800 train 8.100619 (lr=5.4839e-05) (hash(x)=144483776)
+900 val loss 8.0197
+900 val perplexity 3040.3406
+900 train 8.106428 (lr=5.4713e-05) (hash(x)=157916369)
+1000 val loss 7.9809
+1000 val perplexity 2924.5195
+1000 train 8.049354 (lr=5.4553e-05) (hash(x)=154856891)
+1100 val loss 7.9396
+1100 val perplexity 2806.2324
+1100 train 7.836823 (lr=5.4357e-05) (hash(x)=136191502)
+1200 val loss 7.8989
+1200 val perplexity 2694.4019
+1200 train 7.851219 (lr=5.4126e-05) (hash(x)=148021541)
+1300 val loss 7.8440
+1300 val perplexity 2550.3977
+1300 train 7.832902 (lr=5.3860e-05) (hash(x)=146005217)
+1400 val loss 7.8180
+1400 val perplexity 2484.8604
+1400 train 7.762804 (lr=5.3561e-05) (hash(x)=146019502)
+1500 val loss 7.8614
+1500 val perplexity 2595.1614
+1500 train 7.818739 (lr=5.3227e-05) (hash(x)=150127281)
+1600 val loss 7.8539
+1600 val perplexity 2575.7773
+1600 train 7.851043 (lr=5.2860e-05) (hash(x)=154120875)
+1700 val loss 7.8370
+1700 val perplexity 2532.6353
+1700 train 8.066324 (lr=5.2461e-05) (hash(x)=155797680)
+1800 val loss 7.9155
+1800 val perplexity 2739.4517
+1800 train 7.952564 (lr=5.2029e-05) (hash(x)=156809396)
+1900 val loss 7.8837
+1900 val perplexity 2653.5474
+1900 train 7.816091 (lr=5.1565e-05) (hash(x)=144640294)
+2000 val loss 7.8588
+2000 val perplexity 2588.5037
+2000 train 7.866815 (lr=5.1071e-05) (hash(x)=162831106)
+2100 val loss 7.8634
+2100 val perplexity 2600.4246
+2100 train 7.846870 (lr=5.0547e-05) (hash(x)=158239484)
+2200 val loss 7.8814
+2200 val perplexity 2647.5984
+2200 train 7.789166 (lr=4.9993e-05) (hash(x)=140504180)
+2300 val loss 7.8590
+2300 val perplexity 2589.0146
+2300 train 7.847174 (lr=4.9410e-05) (hash(x)=142234024)
+2400 val loss 7.8659
+2400 val perplexity 2606.8308
+2400 train 7.905791 (lr=4.8800e-05) (hash(x)=143091562)
+2500 val loss 7.8647
+2500 val perplexity 2603.7051
+2500 train 7.869723 (lr=4.8162e-05) (hash(x)=149857456)
+2600 val loss 7.8755
+2600 val perplexity 2631.9634
+2600 train 7.938453 (lr=4.7499e-05) (hash(x)=146191551)
+2700 val loss 7.8759
+2700 val perplexity 2633.0557
+2700 train 7.840330 (lr=4.6811e-05) (hash(x)=145375752)
+2800 val loss 7.8727
+2800 val perplexity 2624.5518
+2800 train 7.836661 (lr=4.6099e-05) (hash(x)=151568014)
+2900 val loss 7.8781
+2900 val perplexity 2638.8977
+2900 train 7.828748 (lr=4.5364e-05) (hash(x)=149366597)
+3000 val loss 7.8779
+3000 val perplexity 2638.4260
+3000 train 7.870474 (lr=4.4606e-05) (hash(x)=150464442)
+3100 val loss 7.8758
+3100 val perplexity 2632.8735
+3100 train 7.933112 (lr=4.3828e-05) (hash(x)=182449036)
+3200 val loss 7.8709
+3200 val perplexity 2620.0203
+3200 train 7.874168 (lr=4.3031e-05) (hash(x)=140141286)
+3300 val loss 7.8687
+3300 val perplexity 2614.2461
+3300 train 7.853078 (lr=4.2215e-05) (hash(x)=148099414)
+3400 val loss 7.8560
+3400 val perplexity 2581.3030
+3400 train 7.776361 (lr=4.1381e-05) (hash(x)=142633951)
+3500 val loss 7.8566
+3500 val perplexity 2582.6375
+3500 train 7.821202 (lr=4.0532e-05) (hash(x)=148368965)
+3600 val loss 7.8540
+3600 val perplexity 2575.9565
+3600 train 7.866434 (lr=3.9667e-05) (hash(x)=152372067)
+3700 val loss 7.8593
+3700 val perplexity 2589.6714
+3700 train 7.993324 (lr=3.8789e-05) (hash(x)=168885609)
+3800 val loss 7.8593
+3800 val perplexity 2589.7864
+3800 train 7.760176 (lr=3.7898e-05) (hash(x)=141633734)
+3900 val loss 7.8556
+3900 val perplexity 2580.1855
+3900 train 7.828898 (lr=3.6996e-05) (hash(x)=153141007)
+4000 val loss 7.8517
+4000 val perplexity 2570.2024
+4000 train 7.872583 (lr=3.6085e-05) (hash(x)=160577202)
+4100 val loss 7.8531
+4100 val perplexity 2573.6877
+4100 train 7.917684 (lr=3.5165e-05) (hash(x)=153858169)
+4200 val loss 7.8554
+4200 val perplexity 2579.6443
+4200 train 7.862480 (lr=3.4238e-05) (hash(x)=155889149)
+4300 val loss 7.8516
+4300 val perplexity 2569.9082
+4300 train 7.786407 (lr=3.3305e-05) (hash(x)=152294662)
+4400 val loss 7.8562
+4400 val perplexity 2581.7559
+4400 train 7.770658 (lr=3.2368e-05) (hash(x)=141804386)
+4500 val loss 7.8546
+4500 val perplexity 2577.4409
+4500 train 7.851775 (lr=3.1428e-05) (hash(x)=151095242)
+4600 val loss 7.8571
+4600 val perplexity 2583.9014
+4600 train 7.795098 (lr=3.0486e-05) (hash(x)=156414699)
+4700 val loss 7.8593
+4700 val perplexity 2589.6023
+4700 train 7.897929 (lr=2.9543e-05) (hash(x)=161556686)
+4800 val loss 7.8603
+4800 val perplexity 2592.3215
+4800 train 7.909806 (lr=2.8602e-05) (hash(x)=149000293)
+4900 val loss 7.8604
+4900 val perplexity 2592.5181
+4900 train 7.776464 (lr=2.7663e-05) (hash(x)=154349989)
+5000 val loss 7.8590
+5000 val perplexity 2588.8838
+5000 train 7.718006 (lr=2.6728e-05) (hash(x)=131475967)
+5100 val loss 7.8699
+5100 val perplexity 2617.3655
+5100 train 7.840514 (lr=2.5798e-05) (hash(x)=149717902)
+5200 val loss 7.8545
+5200 val perplexity 2577.4285
+5200 train 7.775805 (lr=2.4874e-05) (hash(x)=151407999)
+5300 val loss 7.9155
+5300 val perplexity 2739.3616
+5300 train 8.021663 (lr=2.3958e-05) (hash(x)=168602728)
+5400 val loss 7.8542
+5400 val perplexity 2576.6531
+5400 train 7.865154 (lr=2.3051e-05) (hash(x)=158344511)
+5500 val loss 7.8489
+5500 val perplexity 2562.9302
+5500 train 7.868498 (lr=2.2155e-05) (hash(x)=148350057)
+5600 val loss 7.8466
+5600 val perplexity 2556.9832
+5600 train 7.876011 (lr=2.1271e-05) (hash(x)=153847323)
+5700 val loss 7.8577
+5700 val perplexity 2585.4592
+5700 train 7.810080 (lr=2.0399e-05) (hash(x)=156607405)
+5800 val loss 7.8789
+5800 val perplexity 2641.0605
+5800 train 7.748581 (lr=1.9542e-05) (hash(x)=145115031)
+5900 val loss 7.8693
+5900 val perplexity 2615.6450
+5900 train 7.735921 (lr=1.8700e-05) (hash(x)=141584622)
+6000 val loss 7.8716
+6000 val perplexity 2621.6274
+6000 train 7.813828 (lr=1.7875e-05) (hash(x)=146613857)
+6100 val loss 7.8564
+6100 val perplexity 2582.3149
+6100 train 7.731911 (lr=1.7068e-05) (hash(x)=144621768)
+6200 val loss 7.8523
+6200 val perplexity 2571.5840
+6200 train 7.774483 (lr=1.6280e-05) (hash(x)=146521760)
+6300 val loss 7.8521
+6300 val perplexity 2571.1362
+6300 train 7.771723 (lr=1.5512e-05) (hash(x)=161378136)
+6400 val loss 7.8516
+6400 val perplexity 2569.8591
+6400 train 7.728492 (lr=1.4766e-05) (hash(x)=141624235)
+6500 val loss 7.8492
+6500 val perplexity 2563.7688
+6500 train 7.779699 (lr=1.4042e-05) (hash(x)=151197095)
+6600 val loss 7.8490
+6600 val perplexity 2563.0476
+6600 train 7.818965 (lr=1.3342e-05) (hash(x)=153269571)
+6700 val loss 7.8527
+6700 val perplexity 2572.7625
+6700 train 7.833497 (lr=1.2666e-05) (hash(x)=146111181)
+6800 val loss 7.8538
+6800 val perplexity 2575.4014
+6800 train 7.856579 (lr=1.2016e-05) (hash(x)=147269760)
+6900 val loss 7.8542
+6900 val perplexity 2576.4565
+6900 train 7.863271 (lr=1.1392e-05) (hash(x)=152912762)
+7000 val loss 7.8477
+7000 val perplexity 2559.8745
+7000 train 7.834257 (lr=1.0795e-05) (hash(x)=165412343)
+7100 val loss 7.8478
+7100 val perplexity 2560.2188
+7100 train 7.900004 (lr=1.0227e-05) (hash(x)=162866028)
+7200 val loss 7.8523
+7200 val perplexity 2571.7458
+7200 train 7.823627 (lr=9.6875e-06) (hash(x)=142998115)
+7300 val loss 7.8481
+7300 val perplexity 2560.9146
+7300 train 7.739837 (lr=9.1780e-06) (hash(x)=145486999)
+7400 val loss 7.8458
+7400 val perplexity 2555.0867
+7400 train 7.842787 (lr=8.6990e-06) (hash(x)=155325873)
+7500 val loss 7.8450
+7500 val perplexity 2552.8750
+7500 train 7.772707 (lr=8.2513e-06) (hash(x)=145131256)
+7600 val loss 7.8443
+7600 val perplexity 2551.1191
+7600 train 7.781463 (lr=7.8355e-06) (hash(x)=144008365)
+7700 val loss 7.8450
+7700 val perplexity 2552.9736
+7700 train 7.735320 (lr=7.4522e-06) (hash(x)=148848532)
+7800 val loss 7.8450
+7800 val perplexity 2552.8750
+7800 train 7.825375 (lr=7.1019e-06) (hash(x)=150391642)
+7900 val loss 7.8448
+7900 val perplexity 2552.5464
+7900 train 7.741000 (lr=6.7852e-06) (hash(x)=152191414)
+8000 val loss 7.8447
+8000 val perplexity 2552.1326
+8000 train 7.915658 (lr=6.5025e-06) (hash(x)=159755587)
+8100 val loss 7.8452
+8100 val perplexity 2553.4827
+8100 train 7.888725 (lr=6.2543e-06) (hash(x)=156664468)
+8200 val loss 7.8448
+8200 val perplexity 2552.4392
+8200 train 7.677240 (lr=6.0408e-06) (hash(x)=139457379)
+8300 val loss 7.8441
+8300 val perplexity 2550.6191
+8300 train 7.742509 (lr=5.8625e-06) (hash(x)=145478564)
+8400 val loss 7.8437
+8400 val perplexity 2549.6560
+8400 train 7.848532 (lr=5.7195e-06) (hash(x)=154982769)
+8500 val loss 7.8429
+8500 val perplexity 2547.6289
+8500 train 7.750079 (lr=5.6121e-06) (hash(x)=145798118)
+8600 val loss 7.8427
+8600 val perplexity 2547.0823
+8600 train 7.798814 (lr=5.5404e-06) (hash(x)=143231551)
+8700 val loss 7.8427
+8700 val perplexity 2547.1333
+8700 train 7.787941 (lr=5.5045e-06) (hash(x)=154780112)
+8749 val loss 7.8426
+8749 val perplexity 2546.8103

lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dae0735ae7ce4b551853272eb12420a2fd0e4ef3b69686b08f7ba792ee02368f
+size 95220594

lr5.5e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1340/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5518fd7d9268e69900fd2cfe36a78f8f616ec84c6e45c0e00e8a66b2fa27a7b3
+size 184163894