Upload folder using huggingface_hub

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -1,12 +1,6 @@
 ---
-title: Coral Streaming
-emoji: 💻
-colorFrom: yellow
-colorTo: yellow
+title: Coral_Streaming
+app_file: app.py
 sdk: gradio
 sdk_version: 5.49.0
-app_file: app.py
-pinned: false
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,191 @@
+from PIL import Image
+import cv2
+import numpy as np
+import gradio as gr
+
+from inference import CoralSegModel, id2label, label2color, create_segmentation_overlay
+model = CoralSegModel()
+
+# ---- helpers ----
+def _safe_read(cap):
+    ok, frame = cap.read()
+    return frame if ok and frame is not None else None
+
+def build_annotations(pred_map: np.ndarray, selected: list[str]) -> list[tuple[np.ndarray, str]]:
+    """Return [(mask,label), ...] where mask is 0/1 float HxW for AnnotatedImage."""
+    if pred_map is None or not selected:
+        return []
+    
+    # Create reverse mapping: label_name -> class_id
+    label2id = {label: int(id_str) for id_str, label in id2label.items()}
+    
+    anns = []
+    for label_name in selected:
+        if label_name not in label2id:
+            continue  # Skip unknown labels
+        
+        class_id = label2id[label_name]  # Convert label name to class ID
+        mask = (pred_map == class_id).astype(np.float32)
+        if mask.sum() > 0:
+            anns.append((mask, label_name))  # Use the label name for display
+    return anns
+    
+# ==============================
+# STREAMING EVENT FUNCTIONS
+# ==============================
+# IMPORTANT: make the event functions themselves generators.
+# Also: include the States as outputs so we can update them every frame.
+def remote_start(url: str, n: int, pred_state, base_state):
+    if not url:
+        return
+    cap = cv2.VideoCapture(url)
+    if not cap.isOpened():
+        return
+    idx = 0
+    try:
+        while True:
+            frame = _safe_read(cap)
+            if frame is None:
+                break
+            if n > 1 and (idx % n) != 0:
+                idx += 1
+                continue
+            pred_map, overlay_rgb, base_rgb = model.predict_map_and_overlay(frame)
+            # yield live image + updated States' *values*
+            yield overlay_rgb, pred_map, base_rgb
+            idx += 1
+    finally:
+        cap.release()
+
+def upload_start(video_file: str, n: int):
+    if not video_file:
+        return
+    cap = cv2.VideoCapture(video_file)
+    if not cap.isOpened():
+        return
+    idx = 0
+    try:
+        while True:
+            ok, frame = cap.read()
+            if not ok or frame is None:
+                break
+            if n > 1 and (idx % n) != 0:
+                idx += 1
+                continue
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            pred_map, overlay_rgb, base_rgb = model.predict_map_and_overlay(frame)
+            yield overlay_rgb, pred_map, base_rgb
+            idx += 1
+    finally:
+        cap.release()
+
+# ==============================
+# SNAPSHOT / TOGGLES (non-streaming)
+# ==============================
+# NOTE: When you pass gr.State as an input, you receive the *value*, not the wrapper.
+def make_snapshot(selected_labels, pred_map, base_rgb, alpha=0.25):
+    if pred_map is None or base_rgb is None:
+        return gr.update()
+    # rebuild overlay to match the live look
+    overlay = create_segmentation_overlay(pred_map, id2label, label2color, Image.fromarray(base_rgb), alpha=alpha)
+    ann = build_annotations(pred_map, selected_labels or [])
+    return (overlay, ann)  # (base_image, [(mask,label), ...])
+
+# ==============================
+# UI
+# ==============================
+with gr.Blocks(title="CoralScapes Streaming Segmentation") as demo:
+    gr.Markdown("# CoralScapes Streaming Segmentation")
+    gr.Markdown(
+        "Left: **live stream** (fast). Right: **snapshot** with **hover labels** and **per-class toggles**."
+    )
+
+    with gr.Tab("Remote Stream (RTSP/HTTP)"):
+        with gr.Row():
+            with gr.Column(scale=2):
+                
+                # States start as None. We'll UPDATE them on every frame by returning them as outputs.
+                pred_state_remote = gr.State(None)  # holds last pred_map (HxW np.uint8)
+                base_state_remote = gr.State(None)  # holds last base_rgb (HxWx3 uint8)
+
+                live_remote = gr.Image(label="Live segmented stream")
+
+                start_btn = gr.Button("Start")
+
+                snap_btn_remote = gr.Button("📸 Snapshot (hover-able)")
+                hover_remote = gr.AnnotatedImage(label="Snapshot (hover to see label)")
+                
+
+            with gr.Column(scale=1):
+                url  = gr.Textbox(label="Stream URL", placeholder="rtsp://user:pass@ip:port/…")
+                skip = gr.Slider(1, 60, value=10, step=1, label="Process every Nth frame")
+
+                toggles_remote = gr.CheckboxGroup(
+                    choices=list(id2label.values()), value=list(id2label.values()),
+                    label="Toggle classes in snapshot",
+                )
+
+            start_btn.click(
+                    remote_start,
+                    inputs=[url, skip, pred_state_remote, base_state_remote],
+                    outputs=[live_remote, pred_state_remote, base_state_remote],
+                    queue=True,   # be explicit; required for generator streaming
+                )
+            
+            snap_btn_remote.click(
+                make_snapshot,
+                inputs=[toggles_remote, pred_state_remote, base_state_remote],
+                outputs=[hover_remote],
+            )
+            toggles_remote.change(
+                make_snapshot,
+                inputs=[toggles_remote, pred_state_remote, base_state_remote],
+                outputs=[hover_remote],
+            )
+
+    with gr.Tab("Upload Video"):
+        with gr.Row():
+            # Left column (now contains toggles, snapshot button, and live output)
+            with gr.Column(scale=2):
+                # States remain in the same column as live_upload
+                pred_state_upload = gr.State(None)
+                base_state_upload = gr.State(None)
+                
+                live_upload = gr.Image(label="Live segmented output")
+                start_btn2 = gr.Button("Process")
+                
+                snap_btn_upload = gr.Button("📸 Snapshot (hover-able)")
+                hover_upload = gr.AnnotatedImage(label="Snapshot (hover to see label)")
+                
+            # Right column (now contains video input and slider)
+            with gr.Column(scale=1):
+                vid_in = gr.Video(sources=["upload"], format="mp4", label="Input Video")
+                skip2 = gr.Slider(1, 5, value=1, step=1, label="Process every Nth frame")
+
+                toggles_upload = gr.CheckboxGroup(
+                    choices=list(id2label.values()), value=list(id2label.values()),
+                    label="Toggle classes in snapshot",
+                )
+                
+        # Event handlers remain the same
+        start_btn2.click(
+            upload_start,
+            inputs=[vid_in, skip2],
+            outputs=[live_upload, pred_state_upload, base_state_upload],
+            queue=True,
+        )
+
+        snap_btn_upload.click(
+            make_snapshot,
+            inputs=[toggles_upload, pred_state_upload, base_state_upload],
+            outputs=[hover_upload],
+        )
+        
+        toggles_upload.change(
+            make_snapshot,
+            inputs=[toggles_upload, pred_state_upload, base_state_upload],
+            outputs=[hover_upload],
+        )
+
+if __name__ == "__main__":
+    demo.queue().launch(share=True)

gradio_demo.py

@@ -0,0 +1,15 @@
+import spaces
+from diffusers import DiffusionPipeline
+
+pipe = DiffusionPipeline.from_pretrained(...)
+pipe.to('cuda')
+
+@spaces.GPU
+def generate(prompt):
+    return pipe(prompt).images
+
+gr.Interface(
+    fn=generate,
+    inputs=gr.Text(),
+    outputs=gr.Gallery(),
+).launch()

inference.py

@@ -0,0 +1,168 @@
+# inference.py
+import torch
+import torch.nn.functional as F
+
+import json
+import urllib.request
+import cv2
+import numpy as np
+from PIL import Image
+from transformers import SegformerImageProcessorFast, SegformerForSemanticSegmentation
+
+id2label = json.load(urllib.request.urlopen(
+    "https://huggingface.co/datasets/EPFL-ECEO/coralscapes/resolve/main/id2label.json"))
+label2color = json.load(urllib.request.urlopen(
+    "https://huggingface.co/datasets/EPFL-ECEO/coralscapes/resolve/main/label2color.json"))
+
+# Load model from HF (swap this with your own if you want)
+HF_MODEL_ID = "EPFL-ECEO/segformer-b5-finetuned-coralscapes-1024-1024"
+
+def create_segmentation_overlay(pred, id2label, label2color, image, alpha=0.25):
+    """
+    Colorizes the segmentation prediction and creates an overlay image.
+
+    Args:
+        pred: The segmentation prediction (numpy array).
+        id2label: Dictionary mapping class IDs to labels.
+        label2color: Dictionary mapping labels to colors.
+        image: The original PIL Image.
+
+    Returns:
+        A PIL Image representing the overlay of the original image and the colorized segmentation mask.
+    """
+    H, W = pred.shape
+    rgb = np.zeros((H, W, 3), dtype=np.uint8)
+
+    # Get unique class IDs present in the prediction
+    unique_classes = np.unique(pred)
+
+    # Create a mapping from class ID to color
+    id2color = {int(id): label2color[label] for id, label in id2label.items()}
+
+    # Define a default color for unknown classes (e.g., black)
+    default_color = [0, 0, 0]
+
+    # Iterate through unique class IDs and colorize the image
+    for class_id in unique_classes:
+        # Get the color for the current class ID, use default_color if not found
+        rgb_c = id2color.get(int(class_id), default_color)
+        # Assign the color to the pixels with the current class ID
+        rgb[pred == class_id] = rgb_c
+
+    mask_rgb = Image.fromarray(rgb)
+
+    # 4) Alpha overlay
+    overlay = Image.blend(image.convert("RGBA"), mask_rgb.convert("RGBA"), alpha=alpha)
+
+    return overlay
+
+def resize_image(image, target_size=1024):
+    """
+    Used to resize the image such that the smaller side equals 1024
+    """
+    h_img, w_img = image.size
+    if h_img < w_img:
+        new_h, new_w = target_size, int(w_img * (target_size / h_img))
+    else:
+        new_h, new_w  = int(h_img * (target_size / w_img)), target_size
+    resized_img = image.resize((new_h, new_w))
+    return resized_img
+
+class CoralSegModel:
+    def __init__(self, device=None):
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+
+        self.processor = SegformerImageProcessorFast.from_pretrained(HF_MODEL_ID)
+
+        self.model = SegformerForSemanticSegmentation.from_pretrained(
+            HF_MODEL_ID,
+            dtype=torch.bfloat16
+        ).to(self.device)
+
+        self.model.eval()
+
+    @torch.inference_mode()
+    def segment_image(self, image, preprocessor, model, crop_size = (1024, 1024), num_classes = 40, batch_size=4) -> np.ndarray:
+        """
+        Batched sliding window inference for improved GPU utilization.
+        """
+        h_crop, w_crop = crop_size
+
+        img = torch.Tensor(np.array(resize_image(image, target_size=1024)).transpose(2, 0, 1)).unsqueeze(0)
+        img = img.to(self.device, torch.bfloat16)
+        _, _, h_img, w_img = img.size()
+
+        h_grids = int(np.round(3/2*h_img/h_crop)) if h_img > h_crop else 1
+        w_grids = int(np.round(3/2*w_img/w_crop)) if w_img > w_crop else 1
+
+        h_stride = int((h_img - h_crop + h_grids -1)/(h_grids -1)) if h_grids > 1 else h_crop
+        w_stride = int((w_img - w_crop + w_grids -1)/(w_grids -1)) if w_grids > 1 else w_crop
+
+        preds = img.new_zeros((1, num_classes, h_img, w_img))
+        count_mat = img.new_zeros((1, 1, h_img, w_img))
+
+        # Collect all crops and their coordinates
+        crops = []
+        coords = []
+        for h_idx in range(h_grids):
+            for w_idx in range(w_grids):
+                y1 = h_idx * h_stride
+                x1 = w_idx * w_stride
+                y2 = min(y1 + h_crop, h_img)
+                x2 = min(x1 + w_crop, w_img)
+                y1 = max(y2 - h_crop, 0)
+                x1 = max(x2 - w_crop, 0)
+                
+                crop_img = img[:, :, y1:y2, x1:x2]
+                crops.append(crop_img)
+                coords.append((x1, x2, y1, y2))
+        
+        # Process crops in batches
+        for i in range(0, len(crops), batch_size):
+            batch_crops = crops[i:i+batch_size]
+            batch_coords = coords[i:i+batch_size]
+            
+            # Stack crops into a batch
+            batch_tensor = torch.cat(batch_crops, dim=0)
+            
+            if preprocessor:
+                inputs = preprocessor(batch_tensor, return_tensors="pt", device=self.device)
+                inputs["pixel_values"] = inputs["pixel_values"].to(self.device, torch.bfloat16)
+            else:
+                inputs = {"pixel_values": batch_tensor}
+            
+            outputs = model(**inputs)
+            
+            # Process each output in the batch
+            for j, (x1, x2, y1, y2) in enumerate(batch_coords):
+                resized_logits = F.interpolate(
+                    outputs.logits[j].unsqueeze(dim=0), 
+                    size=(y2-y1, x2-x1), 
+                    mode="bilinear", 
+                    align_corners=False
+                )
+                preds[:, :, y1:y2, x1:x2] += resized_logits
+                count_mat[:, :, y1:y2, x1:x2] += 1
+
+        assert (count_mat == 0).sum() == 0
+        preds = preds / count_mat
+        preds = preds.argmax(dim=1)
+        preds = F.interpolate(preds.unsqueeze(0).type(torch.uint8), size=image.size[::-1], mode='nearest')
+        label_pred = preds.squeeze().cpu().numpy()
+        return label_pred
+
+    @torch.inference_mode()
+    def predict_map_and_overlay(self, frame_bgr: np.ndarray):
+        """
+        Returns:
+          pred_map:  HxW (uint8/int) with class indices in [0..C-1]
+          overlay:   HxWx3 RGB uint8 (blended color mask over original)
+          rgb:       HxWx3 RGB uint8 original frame (for AnnotatedImage base)
+        """
+        rgb = frame_bgr
+
+        pil = Image.fromarray(rgb)
+        pred = self.segment_image(pil, self.processor, self.model)
+        overlay_rgb = create_segmentation_overlay(pred, id2label, label2color, pil, 0.45)
+        
+        return pred, overlay_rgb, rgb