Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1340/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1340", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1340", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1340, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1340", "n_embd": 128}

attention_kindselective_n_heads2_seed1340/dataloader_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+oid sha256:e95b86432b77c3cf61067c8af4e85b6c7def8ef2dcf5f545b891193eafe80ca3
 size 964

attention_kindselective_n_heads2_seed1340/log2.txt

@@ -1,53 +1,33 @@
 max_steps: 1000
-0 val loss 11.8215
-0 val perplexity 136141.5156
-0 train 11.828733 (lr=7.5000e-07) (hash(x)=108792797)
-50 val loss 10.2379
-50 val perplexity 27942.2988
-100 val loss 9.3592
-100 val perplexity 11605.1504
-100 train 9.345304 (lr=7.5750e-05) (hash(x)=98274409)
-150 val loss 8.3020
-150 val perplexity 4031.8477
-200 val loss 7.7993
-200 val perplexity 2438.8237
-200 train 8.310523 (lr=1.5000e-04) (hash(x)=124454966)
-250 val loss 7.6909
-250 val perplexity 2188.3867
-300 val loss 7.6519
-300 val perplexity 2104.7246
-300 train 7.570929 (lr=1.4486e-04) (hash(x)=100355155)
-350 val loss 7.6552
-350 val perplexity 2111.5752
-400 val loss 7.6471
-400 val perplexity 2094.6055
-400 train 7.469439 (lr=1.3023e-04) (hash(x)=97953247)
-450 val loss 7.6306
-450 val perplexity 2060.3564
-500 val loss 7.6261
-500 val perplexity 2050.9602
-500 train 7.449285 (lr=1.0833e-04) (hash(x)=91758530)
-550 val loss 7.6088
-550 val perplexity 2015.9412
-600 val loss 7.5876
-600 val perplexity 1973.5529
-600 train 7.642048 (lr=8.2500e-05) (hash(x)=111869413)
-650 val loss 7.5746
-650 val perplexity 1948.1482
-700 val loss 7.5607
-700 val perplexity 1921.2594
-700 train 7.497701 (lr=5.6669e-05) (hash(x)=102988617)
-750 val loss 7.5448
-750 val perplexity 1890.8264
-800 val loss 7.5325
-800 val perplexity 1867.7794
-800 train 7.752382 (lr=3.4770e-05) (hash(x)=112812937)
-850 val loss 7.5270
-850 val perplexity 1857.4615
-900 val loss 7.5220
-900 val perplexity 1848.2378
-900 train 7.442432 (lr=2.0138e-05) (hash(x)=100208804)
-950 val loss 7.5186
-950 val perplexity 1841.9471
-999 val loss 7.5168
-999 val perplexity 1838.6222
+0 val loss 11.8210
+0 val perplexity 136085.8281
+0 train 11.833096 (lr=7.5000e-07) (hash(x)=164406924)
+100 val loss 8.9386
+100 val perplexity 7620.5469
+100 train 9.338764 (lr=7.5750e-05) (hash(x)=177407419)
+200 val loss 7.6507
+200 val perplexity 2102.1758
+200 train 7.564567 (lr=1.5000e-04) (hash(x)=144903932)
+300 val loss 7.6006
+300 val perplexity 1999.4196
+300 train 7.959331 (lr=1.4486e-04) (hash(x)=173839165)
+400 val loss 7.5781
+400 val perplexity 1954.9889
+400 train 7.667082 (lr=1.3023e-04) (hash(x)=167734596)
+500 val loss 7.5509
+500 val perplexity 1902.5231
+500 train 7.567864 (lr=1.0833e-04) (hash(x)=153224076)
+600 val loss 7.4864
+600 val perplexity 1783.6138
+600 train 7.387994 (lr=8.2500e-05) (hash(x)=149619098)
+700 val loss 7.4559
+700 val perplexity 1729.9689
+700 train 7.393428 (lr=5.6669e-05) (hash(x)=146539909)
+800 val loss 7.4234
+800 val perplexity 1674.7809
+800 train 7.390232 (lr=3.4770e-05) (hash(x)=153710890)
+900 val loss 7.4136
+900 val perplexity 1658.3834
+900 train 7.366385 (lr=2.0138e-05) (hash(x)=155873620)
+999 val loss 7.3940
+999 val perplexity 1626.1406

attention_kindselective_n_heads2_seed1340/model_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2094dcacf8344e63dadb4e18ca3ae58ab1a24c10b6f4e7d6f3a6d6d1ffaed3fe
+oid sha256:114ef71cdc31dc2732eb2150c9ceb23c7d2dbb582ccf4d912ba07791cbfcf660
 size 38587970

attention_kindselective_n_heads2_seed1340/optimizer_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d69ecb649f12afbc8a21ce3af93dc0599693b4426086bffd21536252414b269b
+oid sha256:49d7cc4f1c71b96c784a6753fe69f1735158c61b3229b6c65f5191eec9cf7e94
 size 70895430