add BLIP features

app.py

@@ -1,6 +1,8 @@
 import numpy as np
+import torch
 from PIL import Image
 from transformers import ViltConfig, ViltProcessor, ViltForQuestionAnswering
+from transformers import BlipProcessor, BlipForQuestionAnswering
 import cv2
 import streamlit as st
 
@@ -13,6 +15,9 @@ model = ViltForQuestionAnswering.from_pretrained("Minqin/carets_vqa_finetuned")
 
 orig_model = ViltForQuestionAnswering.from_pretrained("dandelin/vilt-b32-finetuned-vqa")
 
+blip_processor = BlipProcessor.from_pretrained('Salesforce/blip-vqa-base')
+blip_model = BlipForQuestionAnswering.from_pretrained('Salesforce/blip-vqa-base')
+
 uploaded_file = st.file_uploader("Please upload one image", type=["jpg", "png", "bmp", "jpeg"])
 
 question = st.text_input("Type here one question on the image")
@@ -35,5 +40,15 @@ if uploaded_file is not None:
     orig_logits = orig_outputs.logits
     idx = orig_logits.argmax(-1).item()
     orig_pred = orig_model.config.id2label[idx]
+
+    ## BLIP
+    pixel_values = blip_processor(images=img, return_tensors="pt").pixel_values
+    blip_ques = blip_processor.tokenizer.cls_token + question
+    batch_input_ids = blip_processor(text=blip_ques, add_special_tokens=False).input_ids
+    batch_input_ids = torch.tensor(batch_input_ids)
+
+    generate_ids = blip_model.generate(pixel_values=pixel_values, input_ids=batch_input_ids, max_length=50)
+    blip_output = blip_processor.batch_decode(generate_ids, skip_special_tokens=True)
     st.text(f"Answer of ViLT: {orig_pred}")
+    st.text(f"Answer of BLIP: {blip_output}")
     st.text(f"Answer after fine-tuning: {pred}")