config.json

{
  "activation": "gelu",
  "architectures": [
    "UCEForExpressionPrediction"
  ],
  "d_model": 512,
  "decoder_dropout": 0.1,
  "decoder_layer_dims": [
    1024,
    512,
    512,
    1
  ],
  "dim_feedforward": 2048,
  "dropout": 0.1,
  "dtype": "float32",
  "embedding_dim": 5120,
  "embedding_init_path": null,
  "embedding_reduction": "cls",
  "embedding_requires_grad": false,
  "expansion_factor": 4,
  "input_projection_expansion_factor": 4,
  "input_projection_type": "linear",
  "max_sequence_length": 2048,
  "model_type": "uce",
  "nhead": 4,
  "norm_first": false,
  "num_layers": 8,
  "output_embedding_dim": 512,
  "padding_idx": 0,
  "positional_encoding_type": "sinusoidal",
  "transformers_version": "4.57.1",
  "use_embedding_layer_norm": true,
  "vocab_size": 145469
}