config.json

{"num_blocks": 32, "vocab_size": 32768, "embed_dim": 768, "hidden_dim": 1024, "ffn_dim": 2560, "num_attention_heads": 16, "num_kv_heads": 4, "rope_theta": 100000, "min_lerp_multiplier": 0.1, "temperature_scope": 10}