Spaces:
Running
Running
| import gradio as gr | |
| import torch | |
| from PIL import Image | |
| from transformers import ( | |
| BlipProcessor, BlipForConditionalGeneration, | |
| BlipForQuestionAnswering, | |
| CLIPProcessor, CLIPModel | |
| ) | |
| import numpy as np | |
| # ==================== Model Loading ==================== | |
| print("🔄 Loading models...") | |
| # BLIP Image Captioning Model | |
| caption_processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") | |
| caption_model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") | |
| # BLIP Visual Question Answering Model | |
| vqa_processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") | |
| vqa_model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") | |
| # CLIP Image Classification Model | |
| clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32") | |
| clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32") | |
| print("✅ Models loaded successfully!") | |
| # ==================== Function Definitions ==================== | |
| def generate_caption(image): | |
| """Generate image caption""" | |
| if image is None: | |
| return "❌ Please upload an image first" | |
| try: | |
| # Process image | |
| inputs = caption_processor(image, return_tensors="pt") | |
| # Generate caption | |
| out = caption_model.generate(**inputs, max_length=50) | |
| caption = caption_processor.decode(out[0], skip_special_tokens=True) | |
| return f"📝 Image Caption:\n{caption}" | |
| except Exception as e: | |
| return f"❌ Processing failed: {str(e)}" | |
| def answer_question(image, question): | |
| """Visual Question Answering""" | |
| if image is None: | |
| return "❌ Please upload an image first" | |
| if not question.strip(): | |
| return "❌ Please enter a question" | |
| try: | |
| # Process inputs | |
| inputs = vqa_processor(image, question, return_tensors="pt") | |
| # Generate answer | |
| out = vqa_model.generate(**inputs, max_length=20) | |
| answer = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| return f"❓ Question: {question}\n\n✅ Answer: {answer}" | |
| except Exception as e: | |
| return f"❌ Processing failed: {str(e)}" | |
| def classify_image(image, categories): | |
| """Zero-shot Image Classification""" | |
| if image is None: | |
| return "❌ Please upload an image first" | |
| if not categories.strip(): | |
| return "❌ Please enter categories" | |
| try: | |
| # Parse categories | |
| category_list = [cat.strip() for cat in categories.split(",")] | |
| # Process image and text | |
| inputs = clip_processor( | |
| text=category_list, | |
| images=image, | |
| return_tensors="pt", | |
| padding=True | |
| ) | |
| # Calculate similarity | |
| outputs = clip_model(**inputs) | |
| logits_per_image = outputs.logits_per_image | |
| probs = logits_per_image.softmax(dim=1)[0] | |
| # Format results | |
| results = "🎯 Classification Results:\n\n" | |
| for category, prob in zip(category_list, probs): | |
| percentage = prob.item() * 100 | |
| bar = "█" * int(percentage / 5) | |
| results += f"{category}: {percentage:.2f}% {bar}\n" | |
| return results | |
| except Exception as e: | |
| return f"❌ Processing failed: {str(e)}" | |
| def multimodal_chat(image, message, history): | |
| """Multimodal Chat (Simplified)""" | |
| if image is None: | |
| return history + [[message, "❌ Please upload an image first to start chatting"]] | |
| try: | |
| # Use VQA model to process question | |
| inputs = vqa_processor(image, message, return_tensors="pt") | |
| out = vqa_model.generate(**inputs, max_length=30) | |
| response = vqa_processor.decode(out[0], skip_special_tokens=True) | |
| history.append([message, response]) | |
| return history | |
| except Exception as e: | |
| history.append([message, f"❌ Processing failed: {str(e)}"]) | |
| return history | |
| # ==================== Gradio Interface ==================== | |
| # Custom CSS | |
| custom_css = """ | |
| #title { | |
| text-align: center; | |
| background: linear-gradient(90deg, #667eea 0%, #764ba2 100%); | |
| -webkit-background-clip: text; | |
| -webkit-text-fill-color: transparent; | |
| font-size: 3em; | |
| font-weight: bold; | |
| margin-bottom: 10px; | |
| } | |
| #subtitle { | |
| text-align: center; | |
| color: #666; | |
| font-size: 1.2em; | |
| margin-bottom: 30px; | |
| } | |
| .feature-box { | |
| border: 2px solid #667eea; | |
| border-radius: 10px; | |
| padding: 20px; | |
| margin: 10px 0; | |
| } | |
| """ | |
| with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo: | |
| # Title | |
| gr.HTML('<h1 id="title">🤖 Vision Language AI Demo</h1>') | |
| gr.HTML('<p id="subtitle">Interactive application showcasing multiple vision-language AI capabilities</p>') | |
| # Tabbed Interface | |
| with gr.Tabs(): | |
| # Tab 1: Image Captioning | |
| with gr.Tab("🖼️ Image Captioning"): | |
| gr.Markdown("### Upload an image and AI will generate a description") | |
| with gr.Row(): | |
| with gr.Column(): | |
| caption_image = gr.Image(type="pil", label="Upload Image") | |
| caption_btn = gr.Button("🎨 Generate Caption", variant="primary") | |
| with gr.Column(): | |
| caption_output = gr.Textbox( | |
| label="Generated Caption", | |
| lines=5, | |
| placeholder="Caption will appear here..." | |
| ) | |
| # Examples | |
| gr.Examples( | |
| examples=[ | |
| ["https://images.unsplash.com/photo-1514888286974-6c03e2ca1dba"], | |
| ["https://images.unsplash.com/photo-1506748686214-e9df14d4d9d0"], | |
| ], | |
| inputs=caption_image, | |
| label="📸 Click on examples to try" | |
| ) | |
| caption_btn.click( | |
| fn=generate_caption, | |
| inputs=caption_image, | |
| outputs=caption_output | |
| ) | |
| caption_image.change( | |
| fn=generate_caption, | |
| inputs=caption_image, | |
| outputs=caption_output | |
| ) | |
| # Tab 2: Visual Question Answering | |
| with gr.Tab("🔍 Visual Question Answering"): | |
| gr.Markdown("### Upload an image and ask questions, AI will answer based on the image content") | |
| with gr.Row(): | |
| with gr.Column(): | |
| vqa_image = gr.Image(type="pil", label="Upload Image") | |
| vqa_question = gr.Textbox( | |
| label="Enter Question", | |
| placeholder="e.g., What color is the car? How many people are there?", | |
| lines=2 | |
| ) | |
| vqa_btn = gr.Button("🤔 Get Answer", variant="primary") | |
| with gr.Column(): | |
| vqa_output = gr.Textbox( | |
| label="AI Answer", | |
| lines=6, | |
| placeholder="Answer will appear here..." | |
| ) | |
| # Common question examples | |
| gr.Markdown("**💡 Common Question Examples:**") | |
| gr.Markdown("- What is in the image?\n- What color is...?\n- How many ... are there?\n- Is there a ... in the image?") | |
| vqa_btn.click( | |
| fn=answer_question, | |
| inputs=[vqa_image, vqa_question], | |
| outputs=vqa_output | |
| ) | |
| # Tab 3: Image Classification | |
| with gr.Tab("🏷️ Zero-Shot Classification"): | |
| gr.Markdown("### Define custom categories and AI will classify the image") | |
| with gr.Row(): | |
| with gr.Column(): | |
| classify_image_input = gr.Image(type="pil", label="Upload Image") | |
| classify_categories = gr.Textbox( | |
| label="Categories (comma-separated)", | |
| placeholder="e.g., cat, dog, bird, car, building", | |
| value="cat, dog, bird, car, building", | |
| lines=2 | |
| ) | |
| classify_btn = gr.Button("🎯 Classify", variant="primary") | |
| with gr.Column(): | |
| classify_output = gr.Textbox( | |
| label="Classification Results", | |
| lines=8, | |
| placeholder="Results will appear here..." | |
| ) | |
| gr.Markdown("**💡 Tip:** You can input any categories, the model will calculate similarity between the image and each category") | |
| classify_btn.click( | |
| fn=classify_image, | |
| inputs=[classify_image_input, classify_categories], | |
| outputs=classify_output | |
| ) | |
| # Tab 4: Multimodal Chat | |
| with gr.Tab("💬 Multimodal Chat"): | |
| gr.Markdown("### Upload an image and have a conversation with AI about it") | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| chat_image = gr.Image(type="pil", label="Upload Image") | |
| gr.Markdown("**💡 Conversation Prompts:**") | |
| gr.Markdown("- Describe this image\n- What's in the image?\n- Where is this?\n- What is the main color?") | |
| with gr.Column(scale=2): | |
| chatbot = gr.Chatbot(label="Chat History", height=400) | |
| chat_input = gr.Textbox( | |
| label="Enter Message", | |
| placeholder="Type your question...", | |
| lines=2 | |
| ) | |
| with gr.Row(): | |
| chat_btn = gr.Button("📤 Send", variant="primary") | |
| clear_btn = gr.Button("🗑️ Clear Chat") | |
| chat_btn.click( | |
| fn=multimodal_chat, | |
| inputs=[chat_image, chat_input, chatbot], | |
| outputs=chatbot | |
| ) | |
| chat_input.submit( | |
| fn=multimodal_chat, | |
| inputs=[chat_image, chat_input, chatbot], | |
| outputs=chatbot | |
| ) | |
| clear_btn.click(lambda: [], outputs=chatbot) | |
| # Footer | |
| gr.Markdown("---") | |
| gr.Markdown(""" | |
| ### 📚 About This Application | |
| - **Models**: BLIP (Captioning & VQA) + CLIP (Classification) | |
| - **Framework**: Gradio + Transformers | |
| - **Deployment**: Can be deployed to Hugging Face Spaces | |
| - **Open Source**: All models are open source | |
| ⚡ **Performance Tip**: Use Hugging Face Spaces Zero GPU for significantly faster processing | |
| """) | |
| # Launch application | |
| if __name__ == "__main__": | |
| demo.launch(share=True) |