Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1339/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1339", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1339", "n_embd": 128}

attention_kindselective_n_heads2_seed1339/dataloader_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+oid sha256:e95b86432b77c3cf61067c8af4e85b6c7def8ef2dcf5f545b891193eafe80ca3
 size 964

attention_kindselective_n_heads2_seed1339/log2.txt

@@ -1,53 +1,33 @@
 max_steps: 1000
-0 val loss 11.7471
-0 val perplexity 126386.8516
-0 train 11.738260 (lr=7.5000e-07) (hash(x)=107123132)
-50 val loss 10.5008
-50 val perplexity 36345.9922
-100 val loss 9.5989
-100 val perplexity 14748.7441
-100 train 9.610984 (lr=7.5750e-05) (hash(x)=101193890)
-150 val loss 8.2620
-150 val perplexity 3873.8713
-200 val loss 7.7626
-200 val perplexity 2350.9575
-200 train 7.601669 (lr=1.5000e-04) (hash(x)=95079403)
-250 val loss 7.6888
-250 val perplexity 2183.7668
-300 val loss 7.6583
-300 val perplexity 2118.1362
-300 train 7.509525 (lr=1.4486e-04) (hash(x)=96917853)
-350 val loss 7.6596
-350 val perplexity 2120.8447
-400 val loss 7.6557
-400 val perplexity 2112.6025
-400 train 7.508607 (lr=1.3023e-04) (hash(x)=93672417)
-450 val loss 7.6534
-450 val perplexity 2107.7827
-500 val loss 7.6489
-500 val perplexity 2098.4023
-500 train 7.561789 (lr=1.0833e-04) (hash(x)=98036365)
-550 val loss 7.6362
-550 val perplexity 2071.9031
-600 val loss 7.6217
-600 val perplexity 2042.1052
-600 train 7.674348 (lr=8.2500e-05) (hash(x)=100769188)
-650 val loss 7.6124
-650 val perplexity 2023.1761
-700 val loss 7.6120
-700 val perplexity 2022.3361
-700 train 7.566431 (lr=5.6669e-05) (hash(x)=102854282)
-750 val loss 7.6004
-750 val perplexity 1999.0525
-800 val loss 7.5911
-800 val perplexity 1980.5055
-800 train 7.497270 (lr=3.4770e-05) (hash(x)=99815715)
-850 val loss 7.5890
-850 val perplexity 1976.2594
-900 val loss 7.5804
-900 val perplexity 1959.4106
-900 train 7.345047 (lr=2.0138e-05) (hash(x)=96846928)
-950 val loss 7.5746
-950 val perplexity 1948.0265
-999 val loss 7.5653
-999 val perplexity 1929.9879
+0 val loss 11.7467
+0 val perplexity 126331.5391
+0 train 11.740535 (lr=7.5000e-07) (hash(x)=150724848)
+100 val loss 9.5441
+100 val perplexity 13962.1650
+100 train 9.588784 (lr=7.5750e-05) (hash(x)=149217061)
+200 val loss 7.7467
+200 val perplexity 2313.9080
+200 train 7.728883 (lr=1.5000e-04) (hash(x)=149910534)
+300 val loss 7.6659
+300 val perplexity 2134.2258
+300 train 7.542049 (lr=1.4486e-04) (hash(x)=142185643)
+400 val loss 7.6514
+400 val perplexity 2103.6157
+400 train 7.689255 (lr=1.3023e-04) (hash(x)=148123706)
+500 val loss 7.6465
+500 val perplexity 2093.3313
+500 train 7.502706 (lr=1.0833e-04) (hash(x)=149952383)
+600 val loss 7.6388
+600 val perplexity 2077.1519
+600 train 7.447033 (lr=8.2500e-05) (hash(x)=146678221)
+700 val loss 7.5877
+700 val perplexity 1973.6790
+700 train 7.608317 (lr=5.6669e-05) (hash(x)=156180736)
+800 val loss 7.5683
+800 val perplexity 1935.7953
+800 train 7.559825 (lr=3.4770e-05) (hash(x)=151700982)
+900 val loss 7.5518
+900 val perplexity 1904.2003
+900 train 7.377645 (lr=2.0138e-05) (hash(x)=147288467)
+999 val loss 7.5496
+999 val perplexity 1899.9149

attention_kindselective_n_heads2_seed1339/model_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b9427e25cd8c794fcd6493afec02b0ed8412464d8210605219577d79a8b8c17
+oid sha256:de7e7554d231eb11dab5f290e3b552edec3e9175f4e991f20163430cb3f63a00
 size 38587970

attention_kindselective_n_heads2_seed1339/optimizer_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f956661af1400c1252eb28b994c5ebea1c46edd35433ef521a03303b0bb600d
+oid sha256:2b8cb2649579aecd769fcef623ad66440afeb86aa5511e02ec52d02ff68c8f34
 size 70895430