Upload config.json with huggingface_hub

config.json

@@ -0,0 +1,48 @@
+{
+  "dense_model": {
+    "repo_id": "jacobcd52/ss_d128_f1",
+    "local_path": null
+  },
+  "sparse_model_config": {
+    "n_layer": 2,
+    "d_model": 3072,
+    "n_ctx": 512,
+    "d_head": 16,
+    "d_mlp": 12288,
+    "vocab_size": null,
+    "use_rms_norm": true,
+    "tie_embeddings": false,
+    "use_positional_embeddings": false,
+    "use_bigram_table": false,
+    "use_attention_sinks": true,
+    "activation": "gelu",
+    "dropout": 0.0,
+    "use_bias": true,
+    "use_flash_attention": true
+  },
+  "bridges_config": {
+    "encoder_afrac": 0.25,
+    "coef_nmse": 1.0,
+    "coef_kl_d2s": 1.0,
+    "coef_kl_s2d": 1.0,
+    "coef_ce_sparse": 1.0,
+    "coef_kl_sparse": 1.0
+  },
+  "sparsity_config": {
+    "enable_weight_sparsity": true,
+    "target_l0_fraction": 0.00390625,
+    "sparsity_anneal_start_fraction": 0.01,
+    "sparsity_anneal_end_fraction": 0.5,
+    "anneal_type": "exponential",
+    "min_weights_per_neuron": 4,
+    "enable_activation_sparsity": true,
+    "activation_topk_fraction": 0.25,
+    "activation_sparsity_locations": "attn_in,attn_out,mlp_in,mlp_out,mlp_neuron,attn_v,attn_k,attn_q"
+  },
+  "training_config": {
+    "total_tokens": 2000000000,
+    "batch_size": 64,
+    "dataset_name": "SimpleStories/SimpleStories",
+    "tokenizer_name": "SimpleStories/SimpleStories-1.25M"
+  }
+}