Epoch 0 - Val loss 0.2619

README.md

@@ -0,0 +1,12 @@
+---
+license: apache-2.0
+pipeline_tag: fill-mask
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,21 @@
+{
+  "att_groups": 8,
+  "att_heads": 16,
+  "att_query_groups": 8,
+  "cross_att_type": "sqa",
+  "embed_dim": 512,
+  "ff_activation": "silu",
+  "ff_dim": 1536,
+  "ff_dropout": 0.0,
+  "num_layers": 21,
+  "rope_base": 100000,
+  "self_att_type": "sqa",
+  "seq_len": 8192,
+  "skip_memory_cross_attention": true,
+  "stm_size": 4096,
+  "use_attention_output_bias": false,
+  "use_flash_attention": true,
+  "use_gated": true,
+  "use_gated_attention": true,
+  "vocab_size": 65536
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97d8b17941a9c46850e049680bb774187baa2ed9e0345e7fb14c8bbd2b19e401
+size 194208488