| { | |
| "attention_backend": "sdpa", | |
| "compute_dtype": "torch.bfloat16", | |
| "n_embd": 768, | |
| "n_head": 6, | |
| "n_kv_head": 6, | |
| "n_layer": 12, | |
| "sequence_len": 2048, | |
| "use_activation_checkpointing": true, | |
| "use_vision": true, | |
| "vision_config": { | |
| "image_size": 224, | |
| "mlp_ratio": 4.0, | |
| "n_embd": 768, | |
| "n_head": 12, | |
| "n_layer": 12, | |
| "patch_size": 14, | |
| "rope_theta": 10000.0, | |
| "use_rope": false | |
| }, | |
| "vocab_size": 16386, | |
| "window_pattern": "SSSL" | |
| } |