Upload folder using huggingface_hub

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_10/lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 4, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 8750, "warmup_steps": 500, "group": "wider_is_better_10", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "n_latent_masks", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 61440, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": 2, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 6e-05, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "6e-5_61440_1339", "n_embd": 256}

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/dataloader_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953385078aa3787b69fc6857dfd48b0a2cd2f4d27c6f8892e01211aca53d07f5
+size 964

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/log2.txt

@@ -0,0 +1,267 @@
+max_steps: 8750
+0 val loss 11.2920
+0 val perplexity 80178.2422
+0 train 11.285019 (lr=1.2000e-07) (hash(x)=150724848)
+100 val loss 10.2253
+100 val perplexity 27592.3574
+100 train 10.221909 (lr=1.2120e-05) (hash(x)=149910534)
+200 val loss 9.9783
+200 val perplexity 21554.1348
+200 train 9.981626 (lr=2.4120e-05) (hash(x)=148123706)
+300 val loss 9.7500
+300 val perplexity 17154.0332
+300 train 9.714516 (lr=3.6120e-05) (hash(x)=146678221)
+400 val loss 9.4177
+400 val perplexity 12304.8457
+400 train 9.378237 (lr=4.8120e-05) (hash(x)=151700982)
+500 val loss 9.1259
+500 val perplexity 9190.1885
+500 train 9.181536 (lr=6.0000e-05) (hash(x)=156182087)
+600 val loss 8.8252
+600 val perplexity 6803.8179
+600 train 8.888019 (lr=5.9980e-05) (hash(x)=149318660)
+700 val loss 8.7453
+700 val perplexity 6280.8359
+700 train 8.786180 (lr=5.9922e-05) (hash(x)=150482428)
+800 val loss 8.4069
+800 val perplexity 4477.8125
+800 train 8.354099 (lr=5.9824e-05) (hash(x)=143268605)
+900 val loss 7.9946
+900 val perplexity 2964.8315
+900 train 7.990379 (lr=5.9687e-05) (hash(x)=152322423)
+1000 val loss 9.9248
+1000 val perplexity 20430.8613
+1000 train 9.834566 (lr=5.9512e-05) (hash(x)=147904298)
+1100 val loss 7.7542
+1100 val perplexity 2331.2991
+1100 train 7.878397 (lr=5.9298e-05) (hash(x)=154343147)
+1200 val loss 7.7352
+1200 val perplexity 2287.4661
+1200 train 7.753257 (lr=5.9046e-05) (hash(x)=141843115)
+1300 val loss 7.7147
+1300 val perplexity 2241.1611
+1300 train 7.637745 (lr=5.8757e-05) (hash(x)=145279030)
+1400 val loss 7.8221
+1400 val perplexity 2495.0295
+1400 train 7.844977 (lr=5.8430e-05) (hash(x)=152507639)
+1500 val loss 7.7750
+1500 val perplexity 2380.3945
+1500 train 7.793981 (lr=5.8066e-05) (hash(x)=148473774)
+1600 val loss 7.6820
+1600 val perplexity 2169.0469
+1600 train 7.696146 (lr=5.7666e-05) (hash(x)=151117002)
+1700 val loss 7.6591
+1700 val perplexity 2119.9104
+1700 train 7.581710 (lr=5.7230e-05) (hash(x)=138011335)
+1800 val loss 7.6651
+1800 val perplexity 2132.6978
+1800 train 7.865019 (lr=5.6759e-05) (hash(x)=171180926)
+1900 val loss 7.6559
+1900 val perplexity 2113.1465
+1900 train 7.557738 (lr=5.6253e-05) (hash(x)=141769419)
+2000 val loss 7.7053
+2000 val perplexity 2220.0015
+2000 train 7.661768 (lr=5.5714e-05) (hash(x)=151963443)
+2100 val loss 9.8984
+2100 val perplexity 19898.1152
+2100 train 9.920471 (lr=5.5142e-05) (hash(x)=162947470)
+2200 val loss 8.4872
+2200 val perplexity 4852.0283
+2200 train 8.648426 (lr=5.4537e-05) (hash(x)=154954810)
+2300 val loss 8.5416
+2300 val perplexity 5123.2827
+2300 train 8.576256 (lr=5.3902e-05) (hash(x)=151878111)
+2400 val loss 8.3643
+2400 val perplexity 4291.3022
+2400 train 8.301091 (lr=5.3236e-05) (hash(x)=158661057)
+2500 val loss 8.6220
+2500 val perplexity 5552.6904
+2500 train 8.630486 (lr=5.2541e-05) (hash(x)=150925584)
+2600 val loss 8.1200
+2600 val perplexity 3361.1453
+2600 train 8.050409 (lr=5.1817e-05) (hash(x)=144515755)
+2700 val loss 7.7670
+2700 val perplexity 2361.2847
+2700 train 7.760383 (lr=5.1067e-05) (hash(x)=153109144)
+2800 val loss 7.7642
+2800 val perplexity 2354.7451
+2800 train 7.685761 (lr=5.0290e-05) (hash(x)=151152897)
+2900 val loss 7.7755
+2900 val perplexity 2381.5823
+2900 train 7.724756 (lr=4.9487e-05) (hash(x)=145800210)
+3000 val loss 7.7728
+3000 val perplexity 2375.0500
+3000 train 7.639581 (lr=4.8662e-05) (hash(x)=141997485)
+3100 val loss 7.7659
+3100 val perplexity 2358.8516
+3100 train 7.685941 (lr=4.7813e-05) (hash(x)=154049740)
+3200 val loss 7.7484
+3200 val perplexity 2317.7629
+3200 train 7.735261 (lr=4.6943e-05) (hash(x)=150471842)
+3300 val loss 7.7215
+3300 val perplexity 2256.3220
+3300 train 7.670488 (lr=4.6052e-05) (hash(x)=149048126)
+3400 val loss 7.7029
+3400 val perplexity 2214.7983
+3400 train 7.810497 (lr=4.5143e-05) (hash(x)=161261339)
+3500 val loss 7.7085
+3500 val perplexity 2227.3081
+3500 train 7.714176 (lr=4.4216e-05) (hash(x)=157495564)
+3600 val loss 7.7056
+3600 val perplexity 2220.7944
+3600 train 7.635396 (lr=4.3273e-05) (hash(x)=144352932)
+3700 val loss 7.7015
+3700 val perplexity 2211.6396
+3700 train 7.747025 (lr=4.2315e-05) (hash(x)=149389012)
+3800 val loss 7.7025
+3800 val perplexity 2213.8152
+3800 train 7.647657 (lr=4.1343e-05) (hash(x)=146607620)
+3900 val loss 7.7018
+3900 val perplexity 2212.3696
+3900 train 7.655955 (lr=4.0360e-05) (hash(x)=143754617)
+4000 val loss 7.7066
+4000 val perplexity 2222.8889
+4000 train 7.763909 (lr=3.9365e-05) (hash(x)=156930722)
+4100 val loss 7.7151
+4100 val perplexity 2241.8472
+4100 train 7.579341 (lr=3.8362e-05) (hash(x)=147500519)
+4200 val loss 7.7097
+4200 val perplexity 2229.8638
+4200 train 7.640942 (lr=3.7351e-05) (hash(x)=143232237)
+4300 val loss 7.7154
+4300 val perplexity 2242.7314
+4300 train 7.699511 (lr=3.6333e-05) (hash(x)=146811670)
+4400 val loss 7.7199
+4400 val perplexity 2252.6443
+4400 train 7.778455 (lr=3.5311e-05) (hash(x)=158418746)
+4500 val loss 7.7244
+4500 val perplexity 2262.8223
+4500 train 7.718699 (lr=3.4285e-05) (hash(x)=156695778)
+4600 val loss 7.7186
+4600 val perplexity 2249.7075
+4600 train 7.808847 (lr=3.3257e-05) (hash(x)=147791497)
+4700 val loss 7.7135
+4700 val perplexity 2238.2551
+4700 train 7.606587 (lr=3.2229e-05) (hash(x)=155533088)
+4800 val loss 7.7201
+4800 val perplexity 2253.1235
+4800 train 7.631105 (lr=3.1202e-05) (hash(x)=138350044)
+4900 val loss 7.7145
+4900 val perplexity 2240.6086
+4900 train 7.735612 (lr=3.0178e-05) (hash(x)=143735284)
+5000 val loss 7.7167
+5000 val perplexity 2245.4783
+5000 train 7.697864 (lr=2.9157e-05) (hash(x)=154976463)
+5100 val loss 7.7159
+5100 val perplexity 2243.7263
+5100 train 7.644913 (lr=2.8143e-05) (hash(x)=149894982)
+5200 val loss 7.7189
+5200 val perplexity 2250.4756
+5200 train 7.710912 (lr=2.7135e-05) (hash(x)=159326689)
+5300 val loss 7.7169
+5300 val perplexity 2245.9753
+5300 train 7.700557 (lr=2.6136e-05) (hash(x)=159484800)
+5400 val loss 7.7160
+5400 val perplexity 2244.0632
+5400 train 7.621239 (lr=2.5147e-05) (hash(x)=140385615)
+5500 val loss 7.7170
+5500 val perplexity 2246.1101
+5500 train 7.639513 (lr=2.4169e-05) (hash(x)=148498335)
+5600 val loss 7.7166
+5600 val perplexity 2245.2825
+5600 train 7.590591 (lr=2.3204e-05) (hash(x)=151907614)
+5700 val loss 7.7186
+5700 val perplexity 2249.8726
+5700 train 7.694637 (lr=2.2253e-05) (hash(x)=155192267)
+5800 val loss 7.7194
+5800 val perplexity 2251.5017
+5800 train 7.807099 (lr=2.1318e-05) (hash(x)=153132158)
+5900 val loss 7.7204
+5900 val perplexity 2253.7737
+5900 train 7.767730 (lr=2.0400e-05) (hash(x)=161446764)
+6000 val loss 7.7174
+6000 val perplexity 2247.0432
+6000 train 7.646541 (lr=1.9500e-05) (hash(x)=151512446)
+6100 val loss 7.7192
+6100 val perplexity 2251.0530
+6100 train 7.878604 (lr=1.8620e-05) (hash(x)=188094053)
+6200 val loss 7.7170
+6200 val perplexity 2246.2109
+6200 train 7.656303 (lr=1.7760e-05) (hash(x)=149389789)
+6300 val loss 7.7169
+6300 val perplexity 2246.0266
+6300 train 7.593697 (lr=1.6923e-05) (hash(x)=138212820)
+6400 val loss 7.7162
+6400 val perplexity 2244.3083
+6400 train 7.602308 (lr=1.6108e-05) (hash(x)=146535423)
+6500 val loss 7.7208
+6500 val perplexity 2254.7722
+6500 train 7.634302 (lr=1.5319e-05) (hash(x)=145950843)
+6600 val loss 7.7252
+6600 val perplexity 2264.7998
+6600 train 7.643485 (lr=1.4555e-05) (hash(x)=141162902)
+6700 val loss 7.7363
+6700 val perplexity 2289.8789
+6700 train 7.697646 (lr=1.3817e-05) (hash(x)=153018737)
+6800 val loss 7.7252
+6800 val perplexity 2264.7524
+6800 train 7.728004 (lr=1.3108e-05) (hash(x)=155640155)
+6900 val loss 7.7268
+6900 val perplexity 2268.4248
+6900 train 7.685993 (lr=1.2427e-05) (hash(x)=153722115)
+7000 val loss 7.7246
+7000 val perplexity 2263.3962
+7000 train 7.666291 (lr=1.1777e-05) (hash(x)=146953450)
+7100 val loss 7.7241
+7100 val perplexity 2262.1631
+7100 train 7.632233 (lr=1.1157e-05) (hash(x)=137663885)
+7200 val loss 7.7225
+7200 val perplexity 2258.5190
+7200 train 7.728495 (lr=1.0568e-05) (hash(x)=146172950)
+7300 val loss 7.7224
+7300 val perplexity 2258.3423
+7300 train 7.656596 (lr=1.0012e-05) (hash(x)=150018163)
+7400 val loss 7.7226
+7400 val perplexity 2258.8442
+7400 train 7.672724 (lr=9.4899e-06) (hash(x)=145351166)
+7500 val loss 7.7234
+7500 val perplexity 2260.7214
+7500 train 7.588915 (lr=9.0014e-06) (hash(x)=145292116)
+7600 val loss 7.7229
+7600 val perplexity 2259.4260
+7600 train 7.666094 (lr=8.5478e-06) (hash(x)=150235132)
+7700 val loss 7.7225
+7700 val perplexity 2258.5361
+7700 train 7.663978 (lr=8.1297e-06) (hash(x)=154543455)
+7800 val loss 7.7217
+7800 val perplexity 2256.6824
+7800 train 7.633584 (lr=7.7476e-06) (hash(x)=142456852)
+7900 val loss 7.7226
+7900 val perplexity 2258.8335
+7900 train 7.624061 (lr=7.4021e-06) (hash(x)=147363479)
+8000 val loss 7.7210
+8000 val perplexity 2255.2463
+8000 train 7.752041 (lr=7.0937e-06) (hash(x)=156122973)
+8100 val loss 7.7220
+8100 val perplexity 2257.4036
+8100 train 7.693491 (lr=6.8229e-06) (hash(x)=156153179)
+8200 val loss 7.7231
+8200 val perplexity 2260.0015
+8200 train 7.723278 (lr=6.5900e-06) (hash(x)=146430698)
+8300 val loss 7.7239
+8300 val perplexity 2261.8599
+8300 train 7.654271 (lr=6.3954e-06) (hash(x)=143507257)
+8400 val loss 7.7235
+8400 val perplexity 2260.9175
+8400 train 7.914278 (lr=6.2395e-06) (hash(x)=166272643)
+8500 val loss 7.7267
+8500 val perplexity 2268.0051
+8500 train 7.670963 (lr=6.1223e-06) (hash(x)=143887848)
+8600 val loss 7.7292
+8600 val perplexity 2273.7659
+8600 train 7.830091 (lr=6.0440e-06) (hash(x)=156900341)
+8700 val loss 7.7289
+8700 val perplexity 2273.1243
+8700 train 7.854303 (lr=6.0049e-06) (hash(x)=146417632)
+8749 val loss 7.7292
+8749 val perplexity 2273.8613

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/model_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc6fa078b0d4a04295a1adf9e3ad9c589271092569e1d8490ef85ef84953cf03
+size 95220594

lr6e-5_total_batch_size61440_one_mask_per_head_2_latent_vectors_seed1339/optimizer_08749.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56c58d71e5892e3414a4d3abab24cd66571ade970b3fb1f7af7216aad6a2e022
+size 184163894