fix: update and complete app

README.md

@@ -6,7 +6,7 @@ colorTo: indigo
 sdk: streamlit
 app_file: src/app.py
 python_version: 3.9
-models: [OpenGVLab/Mini-InternVL-Chat-2B-V1-5]
+models: ["google/vit-base-patch16-224-in21k", "bert-base-uncased", "vikenkd/vqa-llm"]
 pinned: false
 ---
 

requirements.txt

@@ -1,15 +1,16 @@
 torch==2.3.1
 torchvision
-cython
-matplotlib
-numpy
-scipy
-pyyaml
-packaging
-pycocotools
-tensorboardx
-h5py
+transformers  
+sentence_transformers
+# cython
+# matplotlib
+# scipy
+# pyyaml
+# packaging
+# pycocotools
+# tensorboardx
+# h5py
 opencv-python
-
+numpy
 streamlit
 pillow # for PIL

src/app.py

@@ -1,9 +1,53 @@
 import streamlit as st
+import numpy as np
+import pandas as pd
+import torch
+
 from PIL import Image
+from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModel
+from infer import InfenceTest
+
+## Read file all of class
+
+
+
+#  Inititalize model
+model_name = "vikenkd/vqa-llm"
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+# Load the model and tokenizer
+model = AutoModel.from_pretrained(model_name)
+model = model.to(device)
+
+# Load tokenize
+visual_feature_extractor_name = "google/vit-base-patch16-224-in21k"
+textual_feature_extractor_name = "bert-base-uncased"
+
+## Text
+tokenizer = AutoTokenizer.from_pretrained(textual_feature_extractor_name)
+text_encoder = AutoModel.from_pretrained(textual_feature_extractor_name)
+for p in text_encoder.parameters():
+    p.requires_grad = False
 
+## Image processor
+image_processor = AutoFeatureExtractor.from_pretrained(visual_feature_extractor_name)
+image_encoder = AutoModel.from_pretrained(visual_feature_extractor_name)
 
+for p in image_encoder.parameters():
+    p.requires_grad = False
 
 
+image_encoder = image_encoder.to(device)
+text_encoder = text_encoder.to(device)
+
+## Initialize class
+infer_encoding = InfenceTest(image_encoder, 
+                             text_encoder, 
+                             tokenizer, 
+                             image_encoder,
+                             device)
+
+# Custom Website App for deploying that model
+
 st.title("📝 Visual Question Answering Project", )
 
 width_head = """
@@ -67,32 +111,25 @@ if image != None:
             with col1:
                 question = st.text_area("Enter text:", placeholder="What is your question?")
             with col2:
-                st.image(image, caption="Uploaded Image.", use_column_width=True)
+                image = st.image(image, caption="Uploaded Image.", use_column_width=True)
+            if not image or  not question:
+                if not image:
+                     st.info("Please upload your image.")
+                elif not question:
+                     st.info("Please add your question.")
+            
             submitted = st.form_submit_button("Submit")     
 
 st.markdown('</div>', unsafe_allow_html=True)
 
+inverse_labels = None
 
-    # if not openai_api_key:
-    #     st.info("Please add your OpenAI API key to continue.")
-    # elif submitted:
-    #     generate_response(text)
-
-
-# if uploaded_file and question and not anthropic_api_key:
-#     st.info("Please add your Anthropic API key to continue.")
-
-# if uploaded_file and question and anthropic_api_key:
-#     article = uploaded_file.read().decode()
-#     prompt = f"""{anthropic.HUMAN_PROMPT} Here's an article:\n\n
-#     {article}\n\n\n\n{question}{anthropic.AI_PROMPT}"""
-
-#     client = anthropic.Client(api_key=anthropic_api_key)
-#     response = client.completions.create(
-#         prompt=prompt,
-#         stop_sequences=[anthropic.HUMAN_PROMPT],
-#         model="claude-v1", #"claude-2" for Claude 2 model
-#         max_tokens_to_sample=100,
-#     )
-#     st.write("### Answer")
-#     st.write(response.completion)
+if submitted:
+    encoding_status = infer_encoding.encoding(quesiton = quesiton, image = image)
+    answer = infer_encoding.infer(model= model, 
+                         inputs_require= encoding_status, 
+                         top_k= 10)
+    st.write("### Answer")
+    st.write(inverse_labels[answer["answer"]])
+    st.write(" - With answer's probability is ")
+    st.write(answer["probs"])

src/infer.py

@@ -0,0 +1,90 @@
+
+from PIL import Image
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+# def infer():
+#     pass
+
+class InfenceTest:
+
+    def __init__(self, 
+                 image_encoder,
+                 text_encoder,
+                 tokenizer,
+                 image_processor,
+                #  batch_size, 
+                 device):
+        
+        self.image_encoder = image_encoder
+        self.text_encoder = text_encoder
+        self.image_processor = image_processor
+        self.tokenizer = tokenizer
+        # self.batch_size = batch_size
+
+        self.device = device
+
+
+    def __len__(self):
+        return len(self.df)
+
+    def encoding(self, question, image):
+
+        image_file = image
+        question = question
+        # full_path = self.type_data + "/images/" + image_file
+        # source_image = os.path.join(os.getcwd(), full_path)
+        image = Image.open(image_file).convert("RGB")
+
+        image_inputs = self.image_processor(image, return_tensors="pt")
+        image_inputs = {k:v.to(self.device) for k,v in image_inputs.items()}
+        image_outputs = self.image_encoder(**image_inputs)
+        image_embedding = image_outputs.pooler_output
+        image_embedding = image_embedding.view(-1)
+        image_embedding = image_embedding.detach()
+
+        text_inputs = self.tokenizer(question, return_tensors="pt")
+        text_inputs = {k:v.to(self.device) for k,v in text_inputs.items()}
+        text_outputs = self.text_encoder(**text_inputs)
+        text_embedding = text_outputs.pooler_output # You can experiment with this or raw CLS embedding below
+        text_embedding = text_embedding.view(-1)
+        text_embedding = text_embedding.detach()
+
+        encoding={}
+        encoding["image_emb"] = image_embedding
+        encoding["question_emb"] = text_embedding
+
+        return encoding
+    
+    def infer(self, model, inputs_require, top_k: int = 10):
+        
+        inputs = {
+            'image_emb':  inputs_require[0],
+            'question_emb': inputs_require[1]
+        }
+
+        with torch.no_grad():
+            outputs = model(**inputs)
+
+        # Apply softmax to get probabilities
+        probabilities = F.softmax(outputs, dim=1)
+        # Get top 10 probabilities and their indices for each example in the batch
+
+        top_probabilities, top_indices = torch.topk(probabilities, k=top_k, dim=1)
+        # print(top_indices.shape)
+        top_indices = top_indices.detach().cpu().numpy()
+
+        probs   = torch.max(outputs.softmax(dim=1), dim=-1)[0].detach().cpu().numpy()
+        outputs = outputs.argmax(-1)
+        logits = outputs.detach().cpu().numpy()
+
+        return {
+                "answer": logits, 
+                "probs": probs, 
+                "topk" : top_indices, 
+                "topk_probs" : top_probabilities
+        }
+
+
+