Add VQA

prismer/model/modules/vit.py

@@ -213,6 +213,8 @@ def load_encoder(name: str, experts: dict, image_resolution: int):
     vision_layers = len([k for k in state_dict.keys() if k.endswith(".attn.in_proj_weight")])
     vision_heads = vision_width // 64
 
+    print(vision_width, vision_patch_size, vision_layers, vision_heads)
+
     ViT = VisionTransformer(input_resolution=image_resolution,
                             patch_size=vision_patch_size,
                             width=vision_width,

prismer_model.py

@@ -103,6 +103,7 @@ class Model:
                 'image_resolution': 480,
                 'prismer_model': model_name,
                 'freeze': 'freeze_vision',
+                'prefix': '',
             }
 
             model = PrismerVQA(config)