| { | |
| "architecture": { | |
| "d_model": 768, | |
| "hidden_dim": 2048, | |
| "max_seq_len": 2048, | |
| "n_heads": 12, | |
| "n_layers": 18, | |
| "rms_norm_eps": 1e-05, | |
| "rope_base": 10000.0, | |
| "vocab_size": 256 | |
| }, | |
| "architecture_name": "ModernByteTransformer", | |
| "attention_type": "causal_sdpa_full_mha_rope", | |
| "chat_format": { | |
| "format": "agent_multiparty", | |
| "room_header": "<|room|> scope=direct members=p0,p1\n<|participants|>\np0=Agent.Ajax\np1=Agent.Steve\n", | |
| "end_sequence": "\n<|end|>\n" | |
| }, | |
| "model_type": "harmonic_byte_transformer", | |
| "parameter_count": 127627008, | |
| "source_checkpoint": "dgx_128m_transformer_agent_sft_ab/agent_multiparty/latest.pt", | |
| "source_pretraining_bytes": 5898240000, | |
| "source_pretraining_step": 90000, | |
| "source_sft_format": "agent_multiparty", | |
| "source_sft_step": 500, | |
| "torch_dtype": "float32", | |
| "vocab_semantics": "token IDs 0-255 map directly to raw byte values" | |
| } | |