Push model using huggingface_hub.

README.md

@@ -1,3 +1,10 @@
----
-license: cc-by-4.0
----
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,43 @@
+{
+  "bridge_args": {
+    "bridge_type": "mlp",
+    "hidden_dim": 2048,
+    "in_dim": 3072,
+    "out_dim": 4096
+  },
+  "embedder": null,
+  "embedder_args": {
+    "causal_embedder": false,
+    "compress_rates": [
+      -8
+    ],
+    "cont_tok": true,
+    "memory_tokens": 0,
+    "n_truncated_layers": 2,
+    "pooling_module": {
+      "pool_type": "mean_pooled_queries",
+      "where": "before"
+    },
+    "rec_tok": true,
+    "train_embedding_mtx": true,
+    "trained_layers": 27
+  },
+  "empty_init": 1,
+  "llms": [],
+  "model_args": {
+    "_sliding_window": null,
+    "dim": 3072,
+    "head_dim": 128,
+    "hidden_dim": 8192,
+    "max_batch_size": 1,
+    "model_type": "transformer",
+    "n_heads": 24,
+    "n_kv_heads": 8,
+    "n_layers": 28,
+    "non_parametric_norm": false,
+    "norm_eps": "1e-05",
+    "rope_theta": 500000.0,
+    "sliding_window": null,
+    "vocab_size": 128256
+  }
+}