Upload folder using huggingface_hub

lr6.5e-5_total_batch_size61440_baseline_seed1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6.5e-5_total_batch_size61440_baseline_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6.5e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6.5e-5_61440_1339", "n_embd": 256}

lr6.5e-5_total_batch_size61440_baseline_seed1339/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6.5e-5_total_batch_size61440_baseline_seed1339/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2808
+0 val perplexity 79287.5234
+0 train 11.289339 (lr=1.3000e-07) (hash(x)=150724848)
+100 val loss 9.9589
+100 val perplexity 21139.1562
+100 train 9.971965 (lr=1.3130e-05) (hash(x)=149910534)
+200 val loss 8.8752
+200 val perplexity 7152.5425
+200 train 8.901247 (lr=2.6130e-05) (hash(x)=148123706)
+300 val loss 7.7552
+300 val perplexity 2333.7058
+300 train 7.595146 (lr=3.9130e-05) (hash(x)=146678221)
+400 val loss 7.4477
+400 val perplexity 1715.8916
+400 train 7.328970 (lr=5.2130e-05) (hash(x)=151700982)
+500 val loss 7.2982
+500 val perplexity 1477.6796
+500 train 7.388100 (lr=6.5000e-05) (hash(x)=156182087)
+600 val loss 7.1546
+600 val perplexity 1279.9194
+600 train 7.121747 (lr=6.4979e-05) (hash(x)=149318660)
+700 val loss 7.0259
+700 val perplexity 1125.4487
+700 train 7.061787 (lr=6.4915e-05) (hash(x)=150482428)
+800 val loss 6.8758
+800 val perplexity 968.5131
+800 train 6.755250 (lr=6.4809e-05) (hash(x)=143268605)
+900 val loss 6.7622
+900 val perplexity 864.5836
+900 train 6.732311 (lr=6.4661e-05) (hash(x)=152322423)
+1000 val loss 6.6775
+1000 val perplexity 794.3677
+1000 train 6.540244 (lr=6.4471e-05) (hash(x)=147904298)
+1100 val loss 6.5963
+1100 val perplexity 732.4147
+1100 train 6.799274 (lr=6.4240e-05) (hash(x)=154343147)
+1200 val loss 6.5033
+1200 val perplexity 667.3455
+1200 train 6.489065 (lr=6.3967e-05) (hash(x)=141843115)
+1300 val loss 6.4411
+1300 val perplexity 627.0664
+1300 train 6.334369 (lr=6.3653e-05) (hash(x)=145279030)
+1400 val loss 6.4122
+1400 val perplexity 609.2511
+1400 train 6.352448 (lr=6.3299e-05) (hash(x)=152507639)
+1500 val loss 6.3784
+1500 val perplexity 588.9749
+1500 train 6.409266 (lr=6.2905e-05) (hash(x)=148473774)
+1600 val loss 6.3291
+1600 val perplexity 560.6451
+1600 train 6.371540 (lr=6.2471e-05) (hash(x)=151117002)
+1700 val loss 6.2817
+1700 val perplexity 534.6735
+1700 train 6.233638 (lr=6.1999e-05) (hash(x)=138011335)
+1800 val loss 6.2460
+1800 val perplexity 515.9589
+1800 train 6.362388 (lr=6.1489e-05) (hash(x)=171180926)
+1900 val loss 6.2139
+1900 val perplexity 499.6649
+1900 train 6.153366 (lr=6.0941e-05) (hash(x)=141769419)
+2000 val loss 6.1834
+2000 val perplexity 484.6555
+2000 train 6.139302 (lr=6.0357e-05) (hash(x)=151963443)
+2100 val loss 6.1804
+2100 val perplexity 483.1944
+2100 train 6.232843 (lr=5.9737e-05) (hash(x)=162947470)
+2200 val loss 6.1303
+2200 val perplexity 459.5791
+2200 train 6.330577 (lr=5.9082e-05) (hash(x)=154954810)
+2300 val loss 6.1120
+2300 val perplexity 451.2220
+2300 train 6.076934 (lr=5.8394e-05) (hash(x)=151878111)
+2400 val loss 6.0893
+2400 val perplexity 441.1075
+2400 train 5.987772 (lr=5.7672e-05) (hash(x)=158661057)
+2500 val loss 6.0560
+2500 val perplexity 426.6549
+2500 train 6.081938 (lr=5.6919e-05) (hash(x)=150925584)
+2600 val loss 6.0375
+2600 val perplexity 418.8568
+2600 train 5.954043 (lr=5.6135e-05) (hash(x)=144515755)
+2700 val loss 6.0193
+2700 val perplexity 411.3037
+2700 train 6.005350 (lr=5.5322e-05) (hash(x)=153109144)
+2800 val loss 5.9928
+2800 val perplexity 400.5512
+2800 train 5.904909 (lr=5.4480e-05) (hash(x)=151152897)
+2900 val loss 5.9748
+2900 val perplexity 393.3928
+2900 train 5.913857 (lr=5.3611e-05) (hash(x)=145800210)
+3000 val loss 5.9707
+3000 val perplexity 391.7803
+3000 train 5.775324 (lr=5.2717e-05) (hash(x)=141997485)
+3100 val loss 5.9364
+3100 val perplexity 378.5611
+3100 train 5.861217 (lr=5.1797e-05) (hash(x)=154049740)
+3200 val loss 5.9192
+3200 val perplexity 372.1196
+3200 train 5.848181 (lr=5.0855e-05) (hash(x)=150471842)
+3300 val loss 5.9123
+3300 val perplexity 369.5414
+3300 train 5.858798 (lr=4.9890e-05) (hash(x)=149048126)
+3400 val loss 5.8896
+3400 val perplexity 361.2741
+3400 train 6.009356 (lr=4.8905e-05) (hash(x)=161261339)
+3500 val loss 5.8712
+3500 val perplexity 354.6580
+3500 train 5.832677 (lr=4.7901e-05) (hash(x)=157495564)
+3600 val loss 5.8610
+3600 val perplexity 351.0731
+3600 train 5.773633 (lr=4.6879e-05) (hash(x)=144352932)
+3700 val loss 5.8416
+3700 val perplexity 344.3308
+3700 train 5.807076 (lr=4.5841e-05) (hash(x)=149389012)
+3800 val loss 5.8233
+3800 val perplexity 338.0715
+3800 train 5.772057 (lr=4.4789e-05) (hash(x)=146607620)
+3900 val loss 5.8136
+3900 val perplexity 334.8128
+3900 train 5.730518 (lr=4.3723e-05) (hash(x)=143754617)
+4000 val loss 5.7954
+4000 val perplexity 328.7905
+4000 train 5.764791 (lr=4.2646e-05) (hash(x)=156930722)
+4100 val loss 5.7830
+4100 val perplexity 324.7282
+4100 train 5.621889 (lr=4.1559e-05) (hash(x)=147500519)
+4200 val loss 5.7710
+4200 val perplexity 320.8541
+4200 train 5.609706 (lr=4.0463e-05) (hash(x)=143232237)
+4300 val loss 5.7555
+4300 val perplexity 315.9383
+4300 train 5.662200 (lr=3.9361e-05) (hash(x)=146811670)
+4400 val loss 5.7413
+4400 val perplexity 311.4615
+4400 train 5.771472 (lr=3.8253e-05) (hash(x)=158418746)
+4500 val loss 5.7379
+4500 val perplexity 310.4192
+4500 train 5.701420 (lr=3.7142e-05) (hash(x)=156695778)
+4600 val loss 5.7261
+4600 val perplexity 306.7764
+4600 train 5.735941 (lr=3.6028e-05) (hash(x)=147791497)
+4700 val loss 5.7117
+4700 val perplexity 302.3958
+4700 train 5.602909 (lr=3.4915e-05) (hash(x)=155533088)
+4800 val loss 5.7034
+4800 val perplexity 299.8935
+4800 train 5.557031 (lr=3.3802e-05) (hash(x)=138350044)
+4900 val loss 5.6905
+4900 val perplexity 296.0479
+4900 train 5.681417 (lr=3.2693e-05) (hash(x)=143735284)
+5000 val loss 5.6788
+5000 val perplexity 292.5863
+5000 train 5.637627 (lr=3.1587e-05) (hash(x)=154976463)
+5100 val loss 5.6731
+5100 val perplexity 290.9319
+5100 train 5.560899 (lr=3.0488e-05) (hash(x)=149894982)
+5200 val loss 5.6656
+5200 val perplexity 288.7518
+5200 train 5.439054 (lr=2.9396e-05) (hash(x)=159326689)
+5300 val loss 5.6555
+5300 val perplexity 285.8546
+5300 train 5.675949 (lr=2.8314e-05) (hash(x)=159484800)
+5400 val loss 5.6430
+5400 val perplexity 282.3147
+5400 train 5.550488 (lr=2.7243e-05) (hash(x)=140385615)
+5500 val loss 5.6348
+5500 val perplexity 279.9986
+5500 train 5.466902 (lr=2.6183e-05) (hash(x)=148498335)
+5600 val loss 5.6356
+5600 val perplexity 280.2216
+5600 train 5.377677 (lr=2.5138e-05) (hash(x)=151907614)
+5700 val loss 5.6251
+5700 val perplexity 277.3112
+5700 train 5.600576 (lr=2.4108e-05) (hash(x)=155192267)
+5800 val loss 5.6129
+5800 val perplexity 273.9254
+5800 train 5.665392 (lr=2.3095e-05) (hash(x)=153132158)
+5900 val loss 5.6077
+5900 val perplexity 272.5055
+5900 train 5.580302 (lr=2.2100e-05) (hash(x)=161446764)
+6000 val loss 5.6054
+6000 val perplexity 271.8824
+6000 train 5.395573 (lr=2.1125e-05) (hash(x)=151512446)
+6100 val loss 5.5922
+6100 val perplexity 268.3358
+6100 train 5.687411 (lr=2.0171e-05) (hash(x)=188094053)
+6200 val loss 5.5832
+6200 val perplexity 265.9203
+6200 train 5.463441 (lr=1.9240e-05) (hash(x)=149389789)
+6300 val loss 5.5808
+6300 val perplexity 265.2888
+6300 train 5.400106 (lr=1.8333e-05) (hash(x)=138212820)
+6400 val loss 5.5769
+6400 val perplexity 264.2455
+6400 train 5.361993 (lr=1.7451e-05) (hash(x)=146535423)
+6500 val loss 5.5669
+6500 val perplexity 261.6269
+6500 train 5.431751 (lr=1.6595e-05) (hash(x)=145950843)
+6600 val loss 5.5635
+6600 val perplexity 260.7408
+6600 train 5.465732 (lr=1.5768e-05) (hash(x)=141162902)
+6700 val loss 5.5611
+6700 val perplexity 260.0999
+6700 train 5.480230 (lr=1.4969e-05) (hash(x)=153018737)
+6800 val loss 5.5518
+6800 val perplexity 257.6882
+6800 train 5.465889 (lr=1.4200e-05) (hash(x)=155640155)
+6900 val loss 5.5490
+6900 val perplexity 256.9789
+6900 train 5.520092 (lr=1.3463e-05) (hash(x)=153722115)
+7000 val loss 5.5422
+7000 val perplexity 255.2353
+7000 train 5.454286 (lr=1.2758e-05) (hash(x)=146953450)
+7100 val loss 5.5413
+7100 val perplexity 255.0107
+7100 train 5.387658 (lr=1.2086e-05) (hash(x)=137663885)
+7200 val loss 5.5331
+7200 val perplexity 252.9342
+7200 train 5.612323 (lr=1.1449e-05) (hash(x)=146172950)
+7300 val loss 5.5300
+7300 val perplexity 252.1325
+7300 train 5.415729 (lr=1.0847e-05) (hash(x)=150018163)
+7400 val loss 5.5268
+7400 val perplexity 251.3314
+7400 train 5.542116 (lr=1.0281e-05) (hash(x)=145351166)
+7500 val loss 5.5268
+7500 val perplexity 251.3389
+7500 train 5.309817 (lr=9.7516e-06) (hash(x)=145292116)
+7600 val loss 5.5190
+7600 val perplexity 249.3767
+7600 train 5.442300 (lr=9.2601e-06) (hash(x)=150235132)
+7700 val loss 5.5156
+7700 val perplexity 248.5269
+7700 train 5.416284 (lr=8.8071e-06) (hash(x)=154543455)
+7800 val loss 5.5140
+7800 val perplexity 248.1372
+7800 train 5.417274 (lr=8.3932e-06) (hash(x)=142456852)
+7900 val loss 5.5132
+7900 val perplexity 247.9402
+7900 train 5.292431 (lr=8.0189e-06) (hash(x)=147363479)
+8000 val loss 5.5069
+8000 val perplexity 246.3896
+8000 train 5.544153 (lr=7.6848e-06) (hash(x)=156122973)
+8100 val loss 5.5044
+8100 val perplexity 245.7778
+8100 train 5.552231 (lr=7.3914e-06) (hash(x)=156153179)
+8200 val loss 5.5031
+8200 val perplexity 245.4429
+8200 train 5.554966 (lr=7.1392e-06) (hash(x)=146430698)
+8300 val loss 5.5015
+8300 val perplexity 245.0607
+8300 train 5.370124 (lr=6.9284e-06) (hash(x)=143507257)
+8400 val loss 5.4967
+8400 val perplexity 243.8756
+8400 train 5.480944 (lr=6.7594e-06) (hash(x)=166272643)
+8500 val loss 5.4954
+8500 val perplexity 243.5574
+8500 train 5.398214 (lr=6.6324e-06) (hash(x)=143887848)
+8600 val loss 5.4944
+8600 val perplexity 243.3177
+8600 train 5.525967 (lr=6.5477e-06) (hash(x)=156900341)
+8700 val loss 5.4891
+8700 val perplexity 242.0348
+8700 train 5.719618 (lr=6.5053e-06) (hash(x)=146417632)
+8749 val loss 5.4890
+8749 val perplexity 242.0133

lr6.5e-5_total_batch_size61440_baseline_seed1339/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:727a4e1a872eb798c4b8b84da8fd13e71bc96ee12ed0344736868abe936569dc
+size 92843394

lr6.5e-5_total_batch_size61440_baseline_seed1339/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dc1de57afd4ec1fc068548125b9cef1a8c73070bb29654213b6b3ac91bef21b
+size 179406214