| { | |
| "vocab_size": 50258, | |
| "embed_dim": 768, | |
| "num_layers": 12, | |
| "num_heads": 12, | |
| "hidden_dim": 3072, | |
| "context_length": 128, | |
| "dropout": 0.1, | |
| "qkv_bias": true, | |
| "vision_dim": 768, | |
| "cross_attention_pos": [ | |
| 3, | |
| 6, | |
| 9 | |
| ], | |
| "num_visual_tokens": 64, | |
| "perceiver_depth": 2, | |
| "perceiver_heads": 8 | |
| } |