config.json

{
  "model_type": "diffurefill",
  "architecture": "bidirectional transformer, masked diffusion (MDLM objective)",
  "vocab_size": 73760,
  "dim": 2048,
  "layers": 18,
  "heads": 16,
  "ff": 5632,
  "max_len": 2048,
  "activation": "swiglu",
  "norm": "rmsnorm (pre-norm)",
  "position": "rope (base 10000)",
  "tied_embeddings": true,
  "mask_token_id": 73759,
  "pad_token_id": 73758,
  "tokenizer": "openbmb/MiniCPM4-0.5B",
  "parameters": 1076000000,
  "training_tokens": 92841363200,
  "training_steps": 200000,
  "weights": "uniform average (SWA) of the 130 merged globals from round 752 to 881 (steps ~181k-200k)"
}