Update app.py

import gradio as gr
import torch
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import os

model_id = "wisdomik/Quilt-Llava-v1.5-7b"
device = "cuda" if torch.cuda.is_available() else "cpu"

processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
model_id,
torch_dtype=torch.float16,
load_in_4bit=True,
device_map="auto"
)

def predict(image, message, history):
if image is not None:
prompt = f"USER: <image>\n{message} ASSISTANT:"
else:
prompt = f"USER: {message} ASSISTANT:"
inputs = processor(prompt, images=image, return_tensors="pt").to(device)
with torch.no_grad():
output = model.generate(**inputs, max_new_tokens=512, temperature=0.7)
response = processor.decode(output[0], skip_special_tokens=True)
response = response.split("ASSISTANT:")[-1].strip()
history.append((message, response))
return history, ""

with gr.Blocks(title="Quilt-LLaVA Patología") as demo:
gr.ChatInterface(predict, multimodal=True)

if __name__ == "__main__":
demo.launch()

app.py

@@ -1,32 +0,0 @@
-import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from llava.model.builder import load_pretrained_model
-from llava.mm_utils import tokenizer_image_token, get_model_name_from_path
-from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
-from llava.conversation import conv_templates
-from PIL import Image
-import torch
-
-model_path = "wisdomik/Quilt-Llava-v1.5-7b"
-tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_base=None, model_name=get_model_name_from_path(model_path))
-
-def predict(image, prompt, history):
-    if image is not None:
-        image_token = DEFAULT_IMAGE_TOKEN
-        prompt = image_token + '\n' + prompt
-    else:
-        prompt = prompt
-    inp = f"{prompt}\nAssistant:"
-    conv = conv_templates["llava_v1"].copy()
-    conv.append_message(conv.roles[0], inp)
-    conv.append_message(conv.roles[1], None)
-    prompt = conv.get_prompt()
-    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
-    with torch.inference_mode():
-        output_ids = model.generate(input_ids, max_new_tokens=512, do_sample=False)
-    response = tokenizer.decode(output_ids[0][input_ids.shape[1]:], skip_special_tokens=True).strip()
-    history.append((prompt, response))
-    return history, ""
-
-iface = gr.ChatInterface(predict, multimodal=True)
-iface.launch()