config.json

{
  "architectures": [
    "DiffusionLlamaLM"
  ],
  "auto_map": {
    "AutoConfig": "configuration_diff_llama.DiffusionLlamaConfig",
    "AutoModel": "modeling_diffusion_llama.DiffusionLlamaLM",
    "AutoModelForCausalLM": "modeling_diff_llama.DiffusionLlamaLM"
  },
  "bias": false,
  "block_size": 2048,
  "condense_ratio": 1,
  "dtype": "float32",
  "eos_token_id": 2,
  "initializer_range": 0.02,
  "intermediate_size": 4096,
  "mask_token_id": 32000,
  "mlp_class": "LLaMAMLP",
  "model_type": "diff_llama",
  "n_embd": 1024,
  "n_head": 16,
  "n_layer": 20,
  "n_query_groups": 16,
  "name": "Diff_LLaMA_336M",
  "norm_class": "FusedRMSNorm",
  "norm_eps": 1e-05,
  "pad_token_id": 0,
  "padded_vocab_size": 32000,
  "padding_multiple": 64,
  "parallel_residual": false,
  "rotary_percentage": 1.0,
  "shared_attention_norm": false,
  "transformers_version": "4.57.3",
  "vocab_size": 32000
}