model switching

eval.py

@@ -0,0 +1,4 @@
+import os
+import time
+
+

future_work/adapters.py

@@ -1,4 +1,3 @@
-from transformers import TextStreamer
 from unsloth import FastVisionModel
 from dotenv import load_dotenv
 import os

future_work/dataset.py

@@ -1,15 +1,14 @@
 from datasets import Dataset
-import torch
 from consts import REASONING_START, REASONING_END, SOLUTION_START, SOLUTION_END 
 
 
 
 def is_numeric_answer(example):
-  try:
-    float(example["answer"])
-    return True
-  except:
-    return False
+    try:
+        float(example["answer"])
+        return True
+    except Exception as e:
+        return f"error: {e}"
 
 def resize_images(example):
     image = example["decoded_image"]

future_work/model.py

@@ -1,5 +1,4 @@
 from unsloth import FastVisionModel
-import torch
 from consts import BASE_MODEL
 
 

gradio_app.py

@@ -1,74 +1,15 @@
-import torch
 import gradio as gr
-from PIL import Image
-from consts import BASE_MODEL
-from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
-from qwen_vl_utils import process_vision_info
+from local_model import query_local
+from remote_model import query_remote, pipe
+import time
 
+def query(image, question, model_name):
+    if model_name == "Local":
+        return query_local(image, question)
+    elif model_name == "Remote":
+        return query_remote(image, question, pipe)
+    return "No model selected"
 
-"""
-Initalize Model
-"""
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-model = Qwen2VLForConditionalGeneration.from_pretrained(BASE_MODEL)
-processor = AutoProcessor.from_pretrained(BASE_MODEL)
-
-
-"""
-Model Function
-"""
-def query(image: Image.Image, question: str):
-    if image is None:
-        return "Upload an image bro."
-
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                {"type": "image", "image": image},
-                {"type": "text", "text": question}
-            ]
-        }
-    ]
-
-    text = processor.apply_chat_template(
-        messages, 
-        tokenize=False, 
-        add_generation_prompt=True
-    )
-    
-    images, video_inputs = process_vision_info(messages)
-
-    inputs = processor(
-        text=text, 
-        images=images, 
-        videos=video_inputs,
-        padding=True, 
-        return_tensors="pt")
-
-    # Generate output
-    generated_ids = model.generate(**inputs, max_new_tokens=256)
-
-    # Trim the input tokens
-    generated_ids_trimmed = [
-        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
-    ]
-    
-    # Decode the output
-    output_text = processor.batch_decode(
-        generated_ids_trimmed, 
-        skip_special_tokens=True, 
-        clean_up_tokenization_spaces=False
-    )
-
-    return output_text[0]
-
-
-
-"""
-Interface
-"""
 
 custom_css = """
 .output-card {
@@ -81,55 +22,49 @@ custom_css = """
 
 with gr.Blocks(theme=gr.themes.Soft(), title="Qwen2-VL Analyst") as app:
     
-    # Header
+    start_time = time.time()
+    
     gr.Markdown(
         r"""
-        ¯\(ツ)/¯ Intelligence: Upload an image and ask a question
+        ¯\_(ツ)_/¯ Intelligence: Upload an image and ask a question
         """
     )
-    
+
     with gr.Row():
-        # Inputs
         with gr.Column(scale=1):
             img_input = gr.Image(type="pil", label="Upload Image", height=400)
-            q_input = gr.Textbox(
-                label="Question", 
-                lines=2
-            )
-            
+            q_input = gr.Textbox(label="Question", lines=2)
             with gr.Row():
                 clear_btn = gr.Button("Clear", variant="secondary")
                 submit_btn = gr.Button("Analyze Image", variant="primary")
-
-        # Output
         with gr.Column(scale=1):
+            with gr.Row():
+                model_dropdown = gr.Dropdown(
+                    label="Select Model", choices=["Local", "Remote"], value="Local"
+                )
             gr.Markdown("Model Analysis:")
-            
+
             with gr.Group(elem_classes="output-card"):
-                output_box = gr.Markdown(
-                    value="Results...", 
-                    line_breaks=True
-                )
+                output_box = gr.Markdown(value="Results...", line_breaks=True)
 
-    # Trigger on Button Click
     submit_btn.click(
-        fn=query, 
-        inputs=[img_input, q_input], 
-        outputs=output_box
+        fn=query, inputs=[img_input, q_input, model_dropdown], outputs=output_box
     )
-    
-    # Trigger on pressing Enter
+
+
+
     q_input.submit(
-        fn=query, 
-        inputs=[img_input, q_input], 
-        outputs=output_box
+        fn=query, inputs=[img_input, q_input, model_dropdown], outputs=output_box
     )
 
-    # Clear button
+
+
     def clear_inputs():
         return None, "", ""
-        
-    clear_btn.click(fn=clear_inputs, inputs=[], outputs=[img_input, q_input, output_box])
+
+    clear_btn.click(
+        fn=clear_inputs, inputs=[], outputs=[img_input, q_input, output_box]
+    )
 
 
 app.launch()

local_model.py

@@ -0,0 +1,62 @@
+import torch
+import gradio as gr
+from PIL import Image
+from consts import BASE_MODEL
+from transformers import AutoProcessor, Qwen2VLForConditionalGeneration
+from qwen_vl_utils import process_vision_info
+import time
+device = "cuda" if torch.cuda.is_available() else "cpu"
+model = Qwen2VLForConditionalGeneration.from_pretrained(BASE_MODEL)
+processor = AutoProcessor.from_pretrained(BASE_MODEL)
+
+
+def query_local(image: Image.Image, question: str):
+    start_time = time.time()
+    if image is None:
+        raise ValueError("Missing image")
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "text", "text": question}
+            ]
+        }
+    ]
+
+    text = processor.apply_chat_template(
+        messages, 
+        tokenize=False, 
+        add_generation_prompt=True
+    )
+    
+    images, video_inputs = process_vision_info(messages)
+
+    inputs = processor(
+        text=text, 
+        images=images, 
+        videos=video_inputs,
+        padding=True, 
+        return_tensors="pt")
+
+    generated_ids = model.generate(**inputs, max_new_tokens=256)
+
+    print("inputs generated")
+    generated_ids_trimmed = [
+        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+    ]
+    print("trimmed")
+    
+    output_text = processor.batch_decode(
+        generated_ids_trimmed, 
+        skip_special_tokens=True, 
+        clean_up_tokenization_spaces=False
+    )
+
+    print("decoded")
+
+    print("local %s --- " % (time.time() - start_time))
+
+    return output_text[0]
+

remote_model.py

@@ -0,0 +1,31 @@
+from huggingface_hub import InferenceClient
+import huggingface_hub
+from consts import BASE_MODEL
+from PIL import Image
+from transformers import pipeline
+import time
+
+
+pipe = pipeline("image-text-to-text", model = BASE_MODEL)
+
+
+def query_remote(image: Image.Image, question: str, pipe):
+    start_time = time.time()
+    if not Image:
+        raise ValueError("Missing image")
+
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image", "image": image},
+                {"type": "text", "text": question}
+            ]
+        }
+    ]
+
+    outputs = pipe(text=messages, return_full_text=False)
+
+    print("remote time %s --- " % (time.time() - start_time))
+
+    return outputs[0]["generated_text"]