| { | |
| "vocab_size": 50304, | |
| "max_seq_len": 1024, | |
| "d_model": 1024, | |
| "n_layers": 24, | |
| "n_heads": 16, | |
| "n_kv_heads": 4, | |
| "ffn_hidden_dim": 2730, | |
| "dropout": 0.0, | |
| "norm_eps": 1e-06, | |
| "rope_theta": 10000.0, | |
| "bias": false, | |
| "tie_weights": true, | |
| "use_flash": true, | |
| "use_qk_norm": true, | |
| "use_diff_attn": true, | |
| "use_mhc": false, | |
| "n_streams": 2, | |
| "mhc_every_n_layers": 1, | |
| "use_xsa": false, | |
| "architecture": "JarvisLM-GPT", | |
| "weights": "ema", | |
| "dtype": "bfloat16", | |
| "parameters": 315758848, | |
| "training_step": 10500, | |
| "source_checkpoint": "last.pt", | |
| "run": "jarvislm-v2-modern-315m-h200-5.5b" | |
| } | |