Upload folder using huggingface_hub

attention_kindself_n_heads2_seed1338/args.json

@@ -1 +1 @@
-{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 80, "total_batch_size": 20480, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_20480_2_1338", "n_embd": 128}
+{"hellaswag": true, "attention_kind": "selective", "log_dir": "wider_is_better_2/attention_kindself_n_heads2_seed1338", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 2, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 1000, "warmup_steps": 200, "group": "wider_is_better_2", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1338, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": true, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 120, "total_batch_size": 30720, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "n_latent_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": true, "mup_enable_coord_check_logging": false, "max_lr": 0.00015, "decay_lr": true, "readout_zero_init": false, "query_zero_init": false, "l1_loss": false, "debugpy": false, "key": "1.5e-4_30720_2_1338", "n_embd": 128}

attention_kindself_n_heads2_seed1338/dataloader_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8513131d2bf732f8f6d97ff82d32c505b2a2f3f9f2247bac315bb09ce0e1f97
+oid sha256:e95b86432b77c3cf61067c8af4e85b6c7def8ef2dcf5f545b891193eafe80ca3
 size 964

attention_kindself_n_heads2_seed1338/log2.txt

@@ -1,53 +1,33 @@
 max_steps: 1000
-0 val loss 11.7388
-0 val perplexity 125337.7812
-0 train 11.765713 (lr=7.5000e-07) (hash(x)=99006591)
-50 val loss 10.4110
-50 val perplexity 33221.8164
-100 val loss 9.7642
-100 val perplexity 17399.4941
-100 train 9.716583 (lr=7.5750e-05) (hash(x)=92897581)
-150 val loss 8.7169
-150 val perplexity 6105.4497
-200 val loss 8.0353
-200 val perplexity 3088.0925
-200 train 7.986736 (lr=1.5000e-04) (hash(x)=102397744)
-250 val loss 7.7502
-250 val perplexity 2322.1006
-300 val loss 7.6778
-300 val perplexity 2159.9612
-300 train 7.528432 (lr=1.4486e-04) (hash(x)=96438273)
-350 val loss 7.6504
-350 val perplexity 2101.4463
-400 val loss 7.6091
-400 val perplexity 2016.5544
-400 train 7.386041 (lr=1.3023e-04) (hash(x)=96121843)
-450 val loss 7.5694
-450 val perplexity 1937.9167
-500 val loss 7.5382
-500 val perplexity 1878.3588
-500 train 7.485167 (lr=1.0833e-04) (hash(x)=102538951)
-550 val loss 7.5006
-550 val perplexity 1809.1721
-600 val loss 7.4816
-600 val perplexity 1775.0961
-600 train 7.258781 (lr=8.2500e-05) (hash(x)=98779605)
-650 val loss 7.4589
-650 val perplexity 1735.1646
-700 val loss 7.4300
-700 val perplexity 1685.8812
-700 train 7.137525 (lr=5.6669e-05) (hash(x)=95523522)
-750 val loss 7.4134
-750 val perplexity 1657.9785
-800 val loss 7.3962
-800 val perplexity 1629.7191
-800 train 7.294775 (lr=3.4770e-05) (hash(x)=110845816)
-850 val loss 7.3761
-850 val perplexity 1597.4272
-900 val loss 7.3491
-900 val perplexity 1554.7263
-900 train 7.459410 (lr=2.0138e-05) (hash(x)=108356411)
-950 val loss 7.3407
-950 val perplexity 1541.7485
-999 val loss 7.3253
-999 val perplexity 1518.2883
+0 val loss 11.7416
+0 val perplexity 125697.3672
+0 train 11.753315 (lr=7.5000e-07) (hash(x)=150327452)
+100 val loss 9.5851
+100 val perplexity 14546.3086
+100 train 9.568905 (lr=7.5750e-05) (hash(x)=166441190)
+200 val loss 7.8255
+200 val perplexity 2503.6711
+200 train 8.030132 (lr=1.5000e-04) (hash(x)=166780046)
+300 val loss 7.6187
+300 val perplexity 2035.9526
+300 train 7.591647 (lr=1.4486e-04) (hash(x)=159835303)
+400 val loss 7.5894
+400 val perplexity 1977.2133
+400 train 7.367747 (lr=1.3023e-04) (hash(x)=155040610)
+500 val loss 7.5263
+500 val perplexity 1856.2716
+500 train 7.213230 (lr=1.0833e-04) (hash(x)=130190460)
+600 val loss 7.4244
+600 val perplexity 1676.3500
+600 train 7.441990 (lr=8.2500e-05) (hash(x)=155504036)
+700 val loss 7.3556
+700 val perplexity 1564.9131
+700 train 7.212329 (lr=5.6669e-05) (hash(x)=137347213)
+800 val loss 7.3140
+800 val perplexity 1501.1636
+800 train 7.167484 (lr=3.4770e-05) (hash(x)=143823248)
+900 val loss 7.2899
+900 val perplexity 1465.3568
+900 train 7.409986 (lr=2.0138e-05) (hash(x)=156260416)
+999 val loss 7.2806
+999 val perplexity 1451.7960

attention_kindself_n_heads2_seed1338/model_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ccf9cd6c291705d92caa7c5e3d82636245985ecf2f619ec076909c2042809ec
+oid sha256:f77fc9e72b2191b389be1fa7fadd26a3aef02852bcfdfa2f38178752a583bc86
 size 38587970

attention_kindself_n_heads2_seed1338/optimizer_00999.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d90a4f03006480f2ea1f9a2fa705578bf9ee93f3ace6925b9f903ed8819fe55
+oid sha256:8ad4bde74c9dbb61b268124600d2c7ddaa9801154371117a4ade9ad0cfa560dd
 size 70895430