| { | |
| "model_type": "diffurefill", | |
| "architecture": "bidirectional transformer, masked diffusion (MDLM objective)", | |
| "vocab_size": 73760, | |
| "dim": 2048, | |
| "layers": 18, | |
| "heads": 16, | |
| "ff": 5632, | |
| "max_len": 2048, | |
| "activation": "swiglu", | |
| "norm": "rmsnorm (pre-norm)", | |
| "position": "rope (base 10000)", | |
| "tied_embeddings": true, | |
| "mask_token_id": 73759, | |
| "pad_token_id": 73758, | |
| "tokenizer": "openbmb/MiniCPM4-0.5B", | |
| "parameters": 1076000000, | |
| "training_tokens": 92841363200, | |
| "training_steps": 200000, | |
| "weights": "uniform average (SWA) of the 130 merged globals from round 752 to 881 (steps ~181k-200k)" | |
| } | |