import streamlit as st import numpy as np import pandas as pd import torch import json from PIL import Image from transformers import AutoTokenizer, AutoFeatureExtractor, AutoModel from infer import InfenceTest ## Read file all of class with open("dataclass/inverse_labels.json", "r") as js: inverse_labels = json.load(js) # Inititalize model model_name = "vikenkd/vqa-llm1" device = "cuda:0" if torch.cuda.is_available() else "cpu" # Load the model and tokenizer model = AutoModel.from_pretrained(model_name) model = model.to(device) # Load tokenize visual_feature_extractor_name = "google/vit-base-patch16-224-in21k" textual_feature_extractor_name = "bert-base-uncased" ## Text tokenizer = AutoTokenizer.from_pretrained(textual_feature_extractor_name) text_encoder = AutoModel.from_pretrained(textual_feature_extractor_name) for p in text_encoder.parameters(): p.requires_grad = False ## Image processor image_processor = AutoFeatureExtractor.from_pretrained(visual_feature_extractor_name) image_encoder = AutoModel.from_pretrained(visual_feature_extractor_name) for p in image_encoder.parameters(): p.requires_grad = False image_encoder = image_encoder.to(device) text_encoder = text_encoder.to(device) ## Initialize class infer_encoding = InfenceTest(image_encoder, text_encoder, tokenizer, image_processor, device) # Custom Website App for deploying that model st.title("📝 Visual Question Answering Project", ) width_head = """ """ st.markdown(width_head, unsafe_allow_html=True) # Custom CSS to create a border border_css = """ """ st.markdown('