Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/dataloader_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+oid sha256:e95b86432b77c3cf61067c8af4e85b6c7def8ef2dcf5f545b891193eafe80ca3
 size 964

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,53 +1,33 @@
 max_steps: 1000
-0 val loss 11.7618
-0 val perplexity 128260.0469
-0 train 11.773180 (lr=7.5000e-07) (hash(x)=99006591)
-50 val loss 10.3640
-50 val perplexity 31698.6367
-100 val loss 9.3084
-100 val perplexity 11030.1504
-100 train 9.241341 (lr=7.5750e-05) (hash(x)=92897581)
-150 val loss 8.2071
-150 val perplexity 3666.7808
-200 val loss 7.7417
-200 val perplexity 2302.3242
-200 train 7.718143 (lr=1.5000e-04) (hash(x)=102397744)
-250 val loss 7.6775
-250 val perplexity 2159.2454
-300 val loss 7.6660
-300 val perplexity 2134.5544
-300 train 7.522061 (lr=1.4486e-04) (hash(x)=96438273)
-350 val loss 7.6725
-350 val perplexity 2148.4053
-400 val loss 7.6628
-400 val perplexity 2127.7815
-400 train 7.450505 (lr=1.3023e-04) (hash(x)=96121843)
-450 val loss 7.6555
-450 val perplexity 2112.1724
-500 val loss 7.6815
-500 val perplexity 2167.9382
-500 train 7.667526 (lr=1.0833e-04) (hash(x)=102538951)
-550 val loss 7.6573
-550 val perplexity 2116.1091
-600 val loss 7.7316
-600 val perplexity 2279.3032
-600 train 7.578350 (lr=8.2500e-05) (hash(x)=98779605)
-650 val loss 7.6675
-650 val perplexity 2137.7324
-700 val loss 7.6600
-700 val perplexity 2121.6589
-700 train 7.399044 (lr=5.6669e-05) (hash(x)=95523522)
-750 val loss 7.6760
-750 val perplexity 2156.0149
-800 val loss 7.6577
-800 val perplexity 2116.9143
-800 train 7.599030 (lr=3.4770e-05) (hash(x)=110845816)
-850 val loss 7.6421
-850 val perplexity 2084.1125
-900 val loss 7.6283
-900 val perplexity 2055.6440
-900 train 7.754482 (lr=2.0138e-05) (hash(x)=108356411)
-950 val loss 7.6223
-950 val perplexity 2043.2234
-999 val loss 7.6118
-999 val perplexity 2021.9426
+0 val loss 11.7630
+0 val perplexity 128408.9922
+0 train 11.769258 (lr=7.5000e-07) (hash(x)=150327452)
+100 val loss 9.2739
+100 val perplexity 10656.4443
+100 train 9.258625 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.7517
+200 val perplexity 2325.5278
+200 train 7.978040 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.6519
+300 val perplexity 2104.7034
+300 train 7.643315 (lr=1.4486e-04) (hash(x)=159835303)
+400 val loss 7.6610
+400 val perplexity 2123.8464
+400 train 7.461915 (lr=1.3023e-04) (hash(x)=155040610)
+500 val loss 7.6780
+500 val perplexity 2160.2671
+500 train 7.390061 (lr=1.0833e-04) (hash(x)=130190460)
+600 val loss 7.6644
+600 val perplexity 2131.1912
+600 train 7.704483 (lr=8.2500e-05) (hash(x)=155504036)
+700 val loss 7.5959
+700 val perplexity 1990.0458
+700 train 7.429086 (lr=5.6669e-05) (hash(x)=137347213)
+800 val loss 7.5658
+800 val perplexity 1931.0613
+800 train 7.440493 (lr=3.4770e-05) (hash(x)=143823248)
+900 val loss 7.5432
+900 val perplexity 1887.8517
+900 train 7.664796 (lr=2.0138e-05) (hash(x)=156260416)
+999 val loss 7.5282
+999 val perplexity 1859.8395

attention_kindselective_n_heads2_seed1338/model_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4532d2e5232b99ad01b07635fafb1824b9cd2e5148cd8b4987d05c1c71e0ad5d
+oid sha256:e042f944bf5db4b04116e7d985ce78f689f7dab71ffad860f815909ae06a5d03
 size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e99d0376c72298505cb2a9b9b287f1a4db0045903102adffcfc879a15c3b8765
+oid sha256:a7c8959f715e25c277c3fee274a30b2ade050ab3c90a025a7084780bee40cec2
 size 70895430