config.json

{
  "dropout": 0.1,
  "hidden_act": "silu",
  "hidden_size": [
    384,
    768,
    768,
    768,
    384
  ],
  "initializer_range": 0.02,
  "input_dim": 1,
  "intermediate_size": [
    1536,
    3072,
    3072,
    3072,
    1536
  ],
  "model_type": "bert",
  "n_heads": [
    4,
    4,
    8,
    8,
    8,
    4,
    4
  ],
  "num_heads": [
    6,
    12,
    12,
    12,
    6
  ],
  "num_latent_tokens": 2,
  "num_layers": [
    1,
    3,
    3,
    3,
    2
  ],
  "num_reg_tokens": 1,
  "quantiles": [
    0.1,
    0.2,
    0.3,
    0.4,
    0.5,
    0.6,
    0.7,
    0.8,
    0.9
  ],
  "scales": [
    1,
    8,
    32,
    8,
    1
  ],
  "transformers_version": "4.57.3",
  "use_latent_tokens": false
}