Update app.py

app.py

@@ -7,8 +7,6 @@ from peft import PeftModel
 import torch.nn as nn
 import whisperx
 import os
-
-
 clip_model_name = "openai/clip-vit-base-patch32"
 phi_model_name  = "microsoft/phi-2"
 tokenizer  = AutoTokenizer.from_pretrained(phi_model_name, trust_remote_code=True)
@@ -18,10 +16,18 @@ IMAGE_TOKEN_ID = 23893 # token for word comment
 device = "cuda" if torch.cuda.is_available() else "cpu"
 clip_embed = 768
 phi_embed  = 2560
+compute_type = "float32"
+audio_batch_size = 16
+
+class SimpleResBlock(nn.Module):
+    def __init__(self, phi_embed):
+        super().__init__()
+        self.pre_norm = nn.LayerNorm(phi_embed)
+        self.proj = nn.Sequential(
+            nn.Linear(phi_embed, phi_embed),
             nn.GELU(),
             nn.Linear(phi_embed, phi_embed)
         )
-
     def forward(self, x):
         x = self.pre_norm(x)
         return x + self.proj(x)
@@ -54,6 +60,9 @@ def model_generate_ans(img=None,img_audio=None,val_q=None):
             val_image_embeds = projection(clip_val_outputs)
             val_image_embeds = resblock(val_image_embeds).to(torch.float16)
             
+            img_token_tensor = torch.tensor(IMAGE_TOKEN_ID).to(device)
+            img_token_embeds = merged_model.model.embed_tokens(img_token_tensor).unsqueeze(0).unsqueeze(0)
+
             val_combined_embeds.append(val_image_embeds)
             val_combined_embeds.append(img_token_embeds)
 
@@ -92,7 +101,6 @@ def model_generate_ans(img=None,img_audio=None,val_q=None):
     return predicted_captions_decoded
     
 
-
 with gr.Blocks() as demo:
 
     gr.Markdown(
@@ -108,7 +116,6 @@ with gr.Blocks() as demo:
             img_input    = gr.Image(label='Image',type="pil")
             img_audio    = gr.Audio(label="Audio Query", sources=['microphone', 'upload'], type='filepath')
             img_question = gr.Text(label ='Text Query')
-
         with gr.Column():
             img_answer   = gr.Text(label ='Answer')