config.json

{
  "model_type": "hourglass_transformer",
  "auto_map": {
    "AutoConfig": "hourglass_transformer.HourglassTransformerConfig",
    "AutoModel": "hourglass_transformer.HourglassTransformerForMaskedLM"
  },
  "activation_function": "gelu",
  "architectures": [
    "HourglassTransformerForMaskedLM"
  ],
  "attn_resampling": false,
  "bias": false,
  "depth": [
    4,
    [
      4,
      4,
      4
    ],
    4
  ],
  "dim": 768,
  "dim_head": 64,
  "heads": 8,
  "inference": false,
  "metadata_dim": 3072,
  "model_type": "hourglass_transformer",
  "norm_out": false,
  "predict_expression_mode": false,
  "predict_seq": true,
  "predict_taxonomy": false,
  "predict_tracks": true,
  "rotary_emb_dim": 32,
  "seq_vocab_size": 11,
  "shorten_factor": [
    8,
    8
  ],
  "sliding_window": [
    512,
    512,
    -1
  ],
  "taxonomy_vocab_size": 2604,
  "torch_dtype": "float32",
  "track_activation_fn": null,
  "track_output_dim": 4,
  "transformers_version": "4.44.2",
  "updown_sample_type": "linear",
  "use_metadata": true,
  "use_taxonomy": false
}