import gradio as gr from transformers import AutoModelForCausalLM, AutoProcessor from peft import PeftModel import torch import warnings from unittest.mock import patch # Ignore warning warnings.filterwarnings("ignore") base_model_id = "microsoft/Florence-2-base" lora_path = "./florence-vqa-lora" device = "mps" if torch.backends.mps.is_available() else "cpu" print(f"Loading Base Model on {device.upper()}...") # Bypass transformers import checker using patch context manager with patch("transformers.dynamic_module_utils.check_imports", return_value=[]): # Fix for transformers 5.x compatibility with Florence-2 from transformers import PretrainedConfig if not hasattr(PretrainedConfig, 'forced_bos_token_id'): PretrainedConfig.forced_bos_token_id = None processor = AutoProcessor.from_pretrained(base_model_id, trust_remote_code=True) base_model = AutoModelForCausalLM.from_pretrained(base_model_id, trust_remote_code=True) print("Merging LoRA Adapters...") model = PeftModel.from_pretrained(base_model, lora_path) model.to(device) model.eval() print("Model loaded successfully!") def predict(image, question): if image is None or not question.strip(): return "⚠️ Please provide both an image and a question." prompt = f" {question}" inputs = processor(text=prompt, images=image, return_tensors="pt").to(device) with torch.no_grad(): generated_ids = model.generate( input_ids=inputs["input_ids"], pixel_values=inputs["pixel_values"], max_new_tokens=50, num_beams=3 ) generated_text = processor.batch_decode(generated_ids, skip_special_tokens=False)[0] parsed_answer = processor.post_process_generation(generated_text, task="", image_size=(image.width, image.height)) return parsed_answer.get('', generated_text) # gradio with gr.Blocks(theme=gr.themes.Soft(), title="Radiology VLM") as demo: gr.Markdown("# 🏥 Generative AI Radiology VLM") gr.Markdown("Ask free-form questions about Medical X-Rays using Microsoft's Florence-2 Vision-Language Model fine-tuned via LoRA.") with gr.Row(): with gr.Column(): image_input = gr.Image(type="pil", label="Upload X-Ray Image") question_input = gr.Textbox(label="Question", placeholder="e.g. What abnormalities are seen in this image?") submit_btn = gr.Button("Analyze Image", variant="primary") with gr.Column(): output_text = gr.Textbox(label="VLM Diagnosis / Answer", lines=5) submit_btn.click(fn=predict, inputs=[image_input, question_input], outputs=output_text) question_input.submit(fn=predict, inputs=[image_input, question_input], outputs=output_text) if __name__ == "__main__": demo.launch(server_name="127.0.0.1", server_port=10987)