import torch from transformers import DetrFeatureExtractor, DetrForObjectDetection from transformers import ViTImageProcessor, ViTModel from PIL import Image, ImageDraw, ImageFont import requests import numpy as np from sklearn.ensemble import IsolationForest import gradio as gr # 1. Load pretrained models detr_feature_extractor = DetrFeatureExtractor.from_pretrained('facebook/detr-resnet-50') detr_model = DetrForObjectDetection.from_pretrained('facebook/detr-resnet-50') vit_processor = ViTImageProcessor.from_pretrained('facebook/dino-vitb16') vit_model = ViTModel.from_pretrained('facebook/dino-vitb16') # COCO classes for DETR object detection COCO_CLASSES = [ 'N/A', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'N/A', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'N/A', 'backpack', 'umbrella', 'N/A', 'N/A', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'N/A', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'N/A', 'dining table', 'N/A', 'N/A', 'toilet', 'N/A', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'N/A', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush' ] # 2. Function: Object detection with DETR def detect_objects(image, confidence_threshold=0.7): inputs = detr_feature_extractor(images=image, return_tensors="pt") outputs = detr_model(**inputs) logits = outputs.logits bboxes = outputs.pred_boxes probas = logits.softmax(-1)[0, :, :-1] # ignore last class (no object) keep = probas.max(-1).values > confidence_threshold scores, labels = probas.max(-1) boxes = bboxes[0, keep] detected_objects = [] for score, label_idx, box in zip(scores[keep], labels[keep], boxes): box = box.detach().cpu().numpy() # Convert box from [center_x, center_y, width, height] normalized format to pixel format w, h = image.size cx, cy, bw, bh = box x_min = int((cx - bw / 2) * w) y_min = int((cy - bh / 2) * h) x_max = int((cx + bw / 2) * w) y_max = int((cy + bh / 2) * h) detected_objects.append({ 'label': COCO_CLASSES[label_idx], 'score': float(score), 'box': [x_min, y_min, x_max, y_max] }) return detected_objects # 3. Function: Scene Recognition with ViT embeddings (dummy classifier) def get_scene_embedding(image): inputs = vit_processor(images=image, return_tensors="pt") outputs = vit_model(**inputs) cls_embedding = outputs.last_hidden_state[:, 0, :].detach().cpu().numpy() # shape (1, hidden_dim) return cls_embedding # For demonstration: A simple heuristic scene classifier with hardcoded labels def classify_scene(embedding): # In practice, train a scene classifier on embeddings. Here dummy logic: avg_value = np.mean(embedding) if avg_value > 0: return "Indoor Scene" else: return "Outdoor Scene" # 4. Simple anomaly detection combining object detection confidence + scene embedding # For demo, anomaly if no object detected OR average detection score below threshold OR unusual scene embedding def is_anomalous(detected_objects, scene_embedding): if len(detected_objects) == 0: return True, "No objects detected - possible anomaly" avg_score = np.mean([obj['score'] for obj in detected_objects]) if avg_score < 0.75: return True, "Low object detection confidence - possible anomaly" # Simple heuristic on embedding mean (you can replace with IsolationForest or more) if np.mean(scene_embedding) < -0.1: return True, "Unusual scene context detected" return False, "Normal" # 5. Draw bounding boxes and labels on image def draw_detections(image, detected_objects, anomaly_flag, anomaly_reason, scene_label): draw = ImageDraw.Draw(image) font = ImageFont.load_default() for obj in detected_objects: box = obj['box'] label = obj['label'] score = obj['score'] draw.rectangle(box, outline="red", width=3) draw.text((box[0], box[1] - 10), f"{label}: {score:.2f}", fill="red", font=font) # Draw anomaly status and scene label draw.text((10, 10), f"Scene: {scene_label}", fill="yellow", font=font) anomaly_text = f"Anomaly Detected: {anomaly_flag} - {anomaly_reason}" draw.text((10, 30), anomaly_text, fill="yellow", font=font) return image # 6. Main processing function for Gradio app def process_image(img): detected_objects = detect_objects(img) scene_emb = get_scene_embedding(img) scene_label = classify_scene(scene_emb) anomaly_flag, anomaly_reason = is_anomalous(detected_objects, scene_emb) img_with_boxes = img.copy() img_with_boxes = draw_detections(img_with_boxes, detected_objects, anomaly_flag, anomaly_reason, scene_label) return img_with_boxes, anomaly_flag, anomaly_reason, scene_label # 7. Build Gradio Interface for Hugging Face Spaces title = "Advanced Vision Pipeline: Object Detection + Scene Recognition + Anomaly Detection" iface = gr.Interface( fn=process_image, inputs=gr.Image(type="pil"), outputs=[ gr.Image(type="pil", label="Output Image with Detections"), gr.Textbox(label="Anomaly Detected?"), gr.Textbox(label="Anomaly Reason"), gr.Textbox(label="Scene Classification"), ], title=title, description="Upload an image to detect objects, recognize scene context, and detect unusual anomalies." ) if __name__ == "__main__": iface.launch()