Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindself_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}

attention_kindself_n_heads2_seed1339/dataloader_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+oid sha256:e95b86432b77c3cf61067c8af4e85b6c7def8ef2dcf5f545b891193eafe80ca3
 size 964

attention_kindself_n_heads2_seed1339/log2.txt

@@ -1,53 +1,33 @@
 max_steps: 1000
-0 val loss 11.7346
-0 val perplexity 124815.0859
-0 train 11.725669 (lr=7.5000e-07) (hash(x)=107123132)
-50 val loss 10.5248
-50 val perplexity 37227.2578
-100 val loss 9.9258
-100 val perplexity 20450.8223
-100 train 9.942008 (lr=7.5750e-05) (hash(x)=101193890)
-150 val loss 8.8761
-150 val perplexity 7159.0596
-200 val loss 8.0385
-200 val perplexity 3097.8174
-200 train 7.907970 (lr=1.5000e-04) (hash(x)=95079403)
-250 val loss 7.7222
-250 val perplexity 2257.9138
-300 val loss 7.6657
-300 val perplexity 2133.9622
-300 train 7.533260 (lr=1.4486e-04) (hash(x)=96917853)
-350 val loss 7.6427
-350 val perplexity 2085.2805
-400 val loss 7.6271
-400 val perplexity 2053.1021
-400 train 7.480326 (lr=1.3023e-04) (hash(x)=93672417)
-450 val loss 7.6040
-450 val perplexity 2006.2183
-500 val loss 7.5738
-500 val perplexity 1946.4379
-500 train 7.482606 (lr=1.0833e-04) (hash(x)=98036365)
-550 val loss 7.5469
-550 val perplexity 1894.9224
-600 val loss 7.5130
-600 val perplexity 1831.6515
-600 train 7.561394 (lr=8.2500e-05) (hash(x)=100769188)
-650 val loss 7.4800
-650 val perplexity 1772.1656
-700 val loss 7.4568
-700 val perplexity 1731.6459
-700 train 7.405756 (lr=5.6669e-05) (hash(x)=102854282)
-750 val loss 7.4347
-750 val perplexity 1693.7745
-800 val loss 7.4204
-800 val perplexity 1669.6633
-800 train 7.337776 (lr=3.4770e-05) (hash(x)=99815715)
-850 val loss 7.4062
-850 val perplexity 1646.2092
-900 val loss 7.3941
-900 val perplexity 1626.3074
-900 train 7.146854 (lr=2.0138e-05) (hash(x)=96846928)
-950 val loss 7.3838
-950 val perplexity 1609.6306
-999 val loss 7.3658
-999 val perplexity 1581.0527
+0 val loss 11.7353
+0 val perplexity 124899.7500
+0 train 11.734357 (lr=7.5000e-07) (hash(x)=150724848)
+100 val loss 9.8470
+100 val perplexity 18902.1074
+100 train 9.889466 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 8.0909
+200 val perplexity 3264.6475
+200 train 8.076604 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.6713
+300 val perplexity 2145.9644
+300 train 7.565024 (lr=1.4486e-04) (hash(x)=142185643)
+400 val loss 7.6198
+400 val perplexity 2038.1362
+400 train 7.653818 (lr=1.3023e-04) (hash(x)=148123706)
+500 val loss 7.5713
+500 val perplexity 1941.6194
+500 train 7.423750 (lr=1.0833e-04) (hash(x)=149952383)
+600 val loss 7.5234
+600 val perplexity 1850.8201
+600 train 7.326363 (lr=8.2500e-05) (hash(x)=146678221)
+700 val loss 7.4437
+700 val perplexity 1709.0602
+700 train 7.472978 (lr=5.6669e-05) (hash(x)=156180736)
+800 val loss 7.3997
+800 val perplexity 1635.4878
+800 train 7.385680 (lr=3.4770e-05) (hash(x)=151700982)
+900 val loss 7.3755
+900 val perplexity 1596.3323
+900 train 7.181541 (lr=2.0138e-05) (hash(x)=147288467)
+999 val loss 7.3566
+999 val perplexity 1566.4429

attention_kindself_n_heads2_seed1339/model_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b063150f8139bc0b5fab6e173ef0a5c9dd780f83ea4fac6cee844f85205f0615
+oid sha256:ee45a8f4f1001570adb6dd07281ec5648500ec5e168e5446943c426c36444643
 size 38587970

attention_kindself_n_heads2_seed1339/optimizer_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4529eb3bfacb30030865096703ae5766e7edd1bd8b151c714defd6bace0deb70
+oid sha256:43f259c149da221e3f0fb335b85aa2f8b9a20de8087a85e7734530ba9827194b
 size 70895430