Upload folder using huggingface_hub

attention_kindselective_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 100, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindselective_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1338", "n_embd": 128}

attention_kindselective_n_heads2_seed1338/dataloader_00999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+size 964

attention_kindselective_n_heads2_seed1338/log2.txt

@@ -1,8 +1,53 @@
-max_steps: 100
+max_steps: 1000
 0 val loss 11.7618
 0 val perplexity 128260.0469
 0 train 11.773180 (lr=7.5000e-07) (hash(x)=99006591)
-50 val loss 10.3708
-50 val perplexity 31913.0859
-99 val loss 9.1780
-99 val perplexity 9681.9316
+50 val loss 10.3640
+50 val perplexity 31698.6367
+100 val loss 9.3084
+100 val perplexity 11030.1504
+100 train 9.241341 (lr=7.5750e-05) (hash(x)=92897581)
+150 val loss 8.2071
+150 val perplexity 3666.7808
+200 val loss 7.7417
+200 val perplexity 2302.3242
+200 train 7.718143 (lr=1.5000e-04) (hash(x)=102397744)
+250 val loss 7.6775
+250 val perplexity 2159.2454
+300 val loss 7.6660
+300 val perplexity 2134.5544
+300 train 7.522061 (lr=1.4486e-04) (hash(x)=96438273)
+350 val loss 7.6725
+350 val perplexity 2148.4053
+400 val loss 7.6628
+400 val perplexity 2127.7815
+400 train 7.450505 (lr=1.3023e-04) (hash(x)=96121843)
+450 val loss 7.6555
+450 val perplexity 2112.1724
+500 val loss 7.6815
+500 val perplexity 2167.9382
+500 train 7.667526 (lr=1.0833e-04) (hash(x)=102538951)
+550 val loss 7.6573
+550 val perplexity 2116.1091
+600 val loss 7.7316
+600 val perplexity 2279.3032
+600 train 7.578350 (lr=8.2500e-05) (hash(x)=98779605)
+650 val loss 7.6675
+650 val perplexity 2137.7324
+700 val loss 7.6600
+700 val perplexity 2121.6589
+700 train 7.399044 (lr=5.6669e-05) (hash(x)=95523522)
+750 val loss 7.6760
+750 val perplexity 2156.0149
+800 val loss 7.6577
+800 val perplexity 2116.9143
+800 train 7.599030 (lr=3.4770e-05) (hash(x)=110845816)
+850 val loss 7.6421
+850 val perplexity 2084.1125
+900 val loss 7.6283
+900 val perplexity 2055.6440
+900 train 7.754482 (lr=2.0138e-05) (hash(x)=108356411)
+950 val loss 7.6223
+950 val perplexity 2043.2234
+999 val loss 7.6118
+999 val perplexity 2021.9426

attention_kindselective_n_heads2_seed1338/model_00999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4532d2e5232b99ad01b07635fafb1824b9cd2e5148cd8b4987d05c1c71e0ad5d
+size 38587970

attention_kindselective_n_heads2_seed1338/optimizer_00999.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e99d0376c72298505cb2a9b9b287f1a4db0045903102adffcfc879a15c3b8765
+size 70895430