import gradio as gr import spaces import torch import numpy as np import cv2 from PIL import Image from transformers import CLIPProcessor, CLIPModel, SegformerImageProcessor, SegformerForSemanticSegmentation # ========================================== # 1. Load Models (CLIP for Vocab, SegFormer for Bias) # ========================================== print("Loading models...") clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch16") clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") seg_model = SegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") seg_processor = SegformerImageProcessor.from_pretrained("nvidia/segformer-b0-finetuned-ade-512-512") print("Models loaded successfully.") # ========================================== # 2. Core Logic Functions # ========================================== def get_clip_saliency(image, text_queries): """Extracts per-patch similarity between image and text queries.""" inputs = clip_processor(text=text_queries, images=image, return_tensors="pt", padding=True) with torch.no_grad(): outputs = clip_model(**inputs) # Get image patch embeddings (Shape: [1, 196, 768] -> 14x14 patches) image_embeds = outputs.image_embeds # This is the CLS token (global) # To get patch-level saliency, we need the last hidden state before projection vision_outputs = clip_model.vision_model(pixel_values=inputs['pixel_values']) patch_embeds = vision_outputs.last_hidden_state[:, 1:, :] # Exclude CLS token patch_embeds = clip_model.visual_projection(patch_embeds) # Project to text space text_embeds = outputs.text_embeds # Shape: [num_texts, 512] # Normalize patch_embeds = patch_embeds / patch_embeds.norm(dim=-1, keepdim=True) text_embeds = text_embeds / text_embeds.norm(dim=-1, keepdim=True) # Calculate similarity matrix similarity = torch.matmul(patch_embeds[0], text_embeds.T) # Shape: [196, num_texts] # Reshape to 14x14 spatial dimensions saliency_maps = similarity.reshape(14, 14, len(text_queries)).cpu().numpy() return saliency_maps def get_foreground_bias(image): """Segments the image to find the main object vs background.""" inputs = seg_processor(images=image, return_tensors="pt") with torch.no_grad(): outputs = seg_model(**inputs) # Upscale segmentation to original image size logits = outputs.logits upscaled_logits = torch.nn.functional.interpolate( logits, size=(image.size[1], image.size[0]), mode="bilinear", align_corners=False ) # Get the most prominent object class (ignoring background class 0 in ADE20k) seg_map = upscaled_logits.argmax(dim=1)[0].cpu().numpy() # Create a binary mask: 1 for foreground objects, 0 for background foreground_mask = (seg_map > 0).astype(np.uint8) * 255 return foreground_mask @spaces.GPU def analyze_transparency(image, text_prompts, opacity_slider): if not image or not text_prompts: return None, None, "Please provide an image and text prompts." image = image.convert("RGB") text_queries = [t.strip() for t in text_prompts.split(",") if t.strip()] # 1. Get Saliency Maps saliency_maps = get_clip_saliency(image, text_queries) # 2. Get Foreground Bias Mask fg_mask = get_foreground_bias(image) fg_pixels = (fg_mask > 0).sum() bg_pixels = (fg_mask == 0).sum() results = [] bias_reports = [] for i, query in enumerate(text_queries): # Extract and normalize the saliency map for this query saliency = saliency_maps[:, :, i] saliency = (saliency - saliency.min()) / (saliency.max() - saliency.min() + 1e-8) # Resize to original image size saliency_resized = cv2.resize(saliency, (image.size[0], image.size[1])) # Generate Heatmap heatmap = cv2.applyColorMap(np.uint8(255 * saliency_resized), cv2.COLORMAP_INFERNO) heatmap = cv2.cvtColor(heatmap, cv2.COLOR_BGR2RGB) # Overlay alpha = opacity_slider overlay = cv2.addWeighted(np.array(image), 1 - alpha, heatmap, alpha, 0) overlay_img = Image.fromarray(overlay) # Calculate Bias # How much attention is on the foreground vs background? total_attention = saliency_resized.sum() + 1e-8 fg_attention = (saliency_resized * (fg_mask > 0)).sum() / total_attention * 100 bg_attention = (saliency_resized * (fg_mask == 0)).sum() / total_attention * 100 # Bias Flag bias_flag = "✅ Normal" if fg_attention > 60 else "⚠️ HIGHLY BIASED" if fg_attention < 40: bias_flag = "🚨 SEVERE CONTEXT BIAS" results.append((overlay_img, f"{query}")) bias_reports.append(f"**{query}**\n- Foreground Focus: `{fg_attention:.1f}%`\n- Background Focus: `{bg_attention:.1f}%`\n- Status: {bias_flag}\n") # Combine into a gallery and a text report bias_report_text = "### 📊 Contextual Bias Report\n" + "\n".join(bias_reports) return results, bias_report_text, bias_report_text # ========================================== # 3. Gradio UI (Handsome & Modern) # ========================================== custom_theme = gr.themes.Soft( primary_hue="emerald", secondary_hue="slate", neutral_hue="slate", font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui"] ).set( body_background_fill="linear-gradient(to bottom right, #0f172a, #1e293b)", body_text_color="#e2e8f0", button_primary_background_fill="#10b981", button_primary_background_fill_hover="#059669", block_border_width="1px", block_border_color="#334155", block_background_fill="#1e293b" ) with gr.Blocks(theme=custom_theme, css="footer {visibility: hidden}") as demo: gr.Markdown( """ # 🧠 Aura: Open-Vocabulary Neural Transparency
Type any concept. Discover exactly where the AI looks, and detect if it is cheating by relying on background context.
""" ) with gr.Row(): with gr.Column(scale=1): input_image = gr.Image(type="pil", label="Input Image", elem_id="input-img") text_input = gr.Textbox( label="Concepts to Search (comma separated)", placeholder="e.g., a dog, a frisbee, green grass, a red car", value="a dog, green grass, a person" ) opacity = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Heatmap Opacity") analyze_btn = gr.Button("🔍 Analyze Transparency", variant="primary") with gr.Column(scale=2): gallery = gr.Gallery( label="Concept Saliency Maps (Where the AI looks for each concept)", show_label=True, columns=2, object_fit="contain", height="500px" ) with gr.Row(): bias_report = gr.Markdown(label="Bias Analysis") analyze_btn.click( fn=analyze_transparency, inputs=[input_image, text_input, opacity], outputs=[gallery, bias_report] ) if __name__ == "__main__": demo.launch()