config.json

{
  "model_type": "frontd",
  "architectures": [
    "FrontD11M"
  ],
  "model_name": "FrontD.11m",
  "vocab_size": 16000,
  "context_length": 1096,
  "hidden_size": 256,
  "num_hidden_layers": 6,
  "num_attention_heads": 16,
  "intermediate_size": 1152,
  "position_embedding_type": "rope",
  "rope_theta": 10000.0,
  "normalization": "rmsnorm",
  "activation_function": "silu",
  "attention_bias": false,
  "mlp_bias": false,
  "tie_word_embeddings": true,
  "dropout": 0.0,
  "bos_token": "<bos>",
  "eos_token": "<eos>",
  "pad_token": "<pad>",
  "unk_token": "<unk>",
  "parameter_target": "~11M",
  "parameter_count": 10996608,
  "dataset": "Plans11/Organized_PreTrain_1k_Context",
  "session_examples": 100000,
  "training_from_scratch": true,
  "license": "mit"
}