| { | |
| "context_len": 120, | |
| "depth": 8, | |
| "dim_conds": [ | |
| 1024, | |
| 1024 | |
| ], | |
| "dim_joint_attn": 1024, | |
| "dim_modalities": [ | |
| 1024, | |
| 1024 | |
| ], | |
| "flash_attn": true, | |
| "image_depth": 6, | |
| "img_channels": 4, | |
| "img_resolution": 32, | |
| "patch_size": 2, | |
| "project_hidden": false, | |
| "qk_rmsnorm": true, | |
| "text_depth": 6, | |
| "vocab_size": 49409 | |
| } |