{
  "model_type": "looplm",
  "architectures": ["LoopLMForCausalLM"],
  "auto_map": {
    "AutoConfig": "configuration_looplm.LoopLMConfig",
    "AutoModelForCausalLM": "modeling_looplm.LoopLMForCausalLM"
  },
  "vocab_size": 50304,
  "d_model": 1024,
  "n_prelude": 4,
  "n_coda": 2,
  "mu_rec": 6,
  "n_q_heads": 16,
  "n_kv_heads": 8,
  "head_dim": 64,
  "qk_norm": true,
  "rope_theta": 10000.0,
  "dense_ffn": 2816,
  "tie_embeddings": true,
  "final_z_loss_coef": 0.0001,
  "use_a_matrix": false,
  "use_input_norm": false,
  "init_std": 0.02,
  "torch_dtype": "float32",
  "transformers_version": "4.40.0"
}