{ "model": { "attention_dropout": 0.0, "drop_path_rate": 0.0, "embedding_dropout": 0.0, "embedding_size": 768, "ffn_dropout": 0.0, "mlp_hidden_size": 1536, "num_attention_heads": 16, "num_cls": 12, "num_layers": 8, "num_outputs": 5000, "positional_damping_factor": 0.1, "residual_scale_init": 1.0, "rms_norm_eps": 1e-06, "rope_base": 10000.0, "rope_fraction": 0.5, "use_absolute_positional_embeddings": false, "use_rope": true } }