config.json

{
  "library_name": "pytorch",
  "model_type": "deepx-embedding",
  "architectures": ["DeepXEmbedding"],
  "hidden_size": 1536,
  "vocab_size": 262144,
  "max_position_embeddings": 8192,
  "num_attention_heads": 8,
  "num_hidden_layers": 35,
  "total_params": 889000000,
  "trainable_params": 486000000,
  "embedding_dimension": 1536,
  "matryoshka_dims": [256, 512, 768, 1024, 1536],
  "attention_type": "gdn2_linear",
  "weight_sharing": "hyperloop"
}