Upload folder using huggingface_hub

mha_conv_coord_check_4/mha_conv_seed_1339/args.json

@@ -0,0 +1 @@
+{"hellaswag": true, "attention_kind": "dense", "log_dir": "mha_conv_coord_check_4/mha_conv_seed_1339", "resume_checkpoint": null, "resume_optimizer": false, "add_a_head": false, "add_head_to_start": true, "new_head_init": "normal", "n_heads": 8, "n_embd": 256, "head_dim": 32, "protect_bos_token": true, "prevent_from_masking_myself": true, "max_steps": 20, "warmup_steps": 250, "group": "mha_conv_coord_check_4", "use_wandb": true, "kill_self_after_run": false, "random_seed": 1339, "memory_penalty_epsilon": 0.1, "selection_head_linear_combo": "none", "selection_head_linear_combo_scale": 1.0, "disable_selection_head_linear_combo_bias": false, "assert_latent_matches_no_head": false, "n_latent_masks": null, "init_latent_masks_to_identity": false, "init_latent_masks_to_inverse": false, "latent_mask_scale": null, "latent_mask_runtime_multiplier": null, "latent_mask_sigmoid": false, "S_layernorm": false, "one_head_per_latent_mask": false, "att_conv": false, "att_conv_init": "none", "att_conv_scale": 1.0, "att_conv_weight_decay": false, "protection_kind": "none", "leaky_relu_alpha": null, "leaky_relu_bias": null, "use_compile": false, "use_mini_model": false, "upload_to_hf": true, "seq_len": 256, "batch_size": 16, "total_batch_size": 8192, "protection_head_scaling_factor": 1.0, "protection_head_bias": 0.0, "n_sliced_masks": null, "mask_layernorm": false, "residual_attention_masks": false, "compute_base_shapes": false, "base_shapes_savefile": null, "mup": true, "disable_selection": false, "mup_enable_coord_check_logging": true, "max_lr": 0.003, "decay_lr": false, "readout_zero_init": false, "query_zero_init": false, "mup_zero_init": true, "l1_loss": false, "debugpy": false, "key": "mha_conv_seed_1339", "latent_mask_precision": "bfloat16", "att_conv_precision": "bfloat16", "attn_precision": "bfloat16", "profile_kind": "none", "dense_attention_kind": "mha", "head_dim_value": 32, "sdpa_iter_size": null, "stabilize_attn_scores": true, "override_use_sdpa": false, "autocast_precision": "bfloat16", "simulate_micro_bs": null, "simulate_micro_bs_2": null, "ckpt_attn": false, "a_producer_kind": "mha_conv", "qkv_producer_kind": "mha", "av_combiner_kind": "linear", "c_proj_scale_init": null}

mha_conv_coord_check_4/mha_conv_seed_1339/dataloader_00019.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f8dd5cbb1553ae23ce415de905eefc7aa408e5ef5761c147c9b0e80bf5e76dc
+size 964

mha_conv_coord_check_4/mha_conv_seed_1339/log2.txt

@@ -0,0 +1,7 @@
+max_steps: 20
+0 val loss 11.3227
+0 val perplexity 82675.5781
+0 train 11.308193 (lr=3.0000e-03) (hash(x)=20295198)
+10 train 9.115982 (lr=3.0000e-03) (hash(x)=20294776)
+19 val loss 8.5394
+19 val perplexity 5112.2329

mha_conv_coord_check_4/mha_conv_seed_1339/model_00019.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70b732cb94b12450d39e69f0cba06f33e1d2eb4d173aa2cfb7ecae76ad92073e
+size 92859506

mha_conv_coord_check_4/mha_conv_seed_1339/optimizer_00019.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:871db5e0846cf4e11ef0f0c12364dc392dd2eabdc81c05322b3c7c67ac845c69
+size 179433270