Update app.py

app.py

@@ -29,20 +29,22 @@ from qwen_vl_utils import process_vision_info
 import gradio as gr
 from PIL import Image
 import torch
-print(torch.cuda.memory_summary())
+
 
 
 # Create a configuration for quantization
 quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,   # Set to True for 4-bit quantization
-    bnb_4bit_compute_dtype="float16",  # Use float16 for faster computations
-    bnb_4bit_use_double_quant=True,    # Optional: Double quantization for memory savings
-    bnb_4bit_quant_type="nf4",         # NormalFloat4 (nf4) is better for performance
+    load_in_4bit=True,   
+    bnb_4bit_compute_dtype="float16",  
+    bnb_4bit_use_double_quant=True,    
+    bnb_4bit_quant_type="nf4",         
 )
 
 # Load the model and processor
 model = Qwen2VLForConditionalGeneration.from_pretrained(
-    "Qwen/QVQ-72B-Preview", device_map="auto", quantization_config=quantization_config,
+    "Qwen/QVQ-72B-Preview", device_map="auto", 
+    quantization_config=quantization_config,
+    offload_folder="offload",
 )
 processor = AutoProcessor.from_pretrained("Qwen/QVQ-72B-Preview")
 
@@ -89,7 +91,7 @@ def process_image_and_question(image, question):
     output_text = processor.batch_decode(
         generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
     )
-    
+    print(output_text[0] if output_text else "No output generated.")
     return output_text[0] if output_text else "No output generated."
 
 # Define the Gradio interface
@@ -115,4 +117,5 @@ with gr.Blocks() as demo:
 # Launch the interface
 demo.launch()
 
+print(torch.cuda.memory_summary())