"""P2Echo: text-prompted multi-view echocardiography segmentation demo. Upload an echocardiography frame, pick the view and the cardiac structures to segment, and P2Echo returns a colour-coded segmentation mask following the structured text prompt built from your selection. """ import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # noqa: E402 (must precede torch) import torch # noqa: E402 import gradio as gr # noqa: E402 import numpy as np # noqa: E402 from PIL import Image # noqa: E402 from p2echo import P2Echo, TextEncoder # noqa: E402 from p2echo.data import normalize_image # noqa: E402 from p2echo.prompts import VIEW_STRUCTURES, structure_prompt # noqa: E402 DEVICE = "cuda" IMAGE_SIZE = 256 TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B" CHECKPOINT = "p2echo.pth" # shipped in the Space repo # ---------------------------------------------------------------- model load text_encoder = TextEncoder(TEXT_MODEL).to(DEVICE) model = P2Echo(text_dim=text_encoder.dim).to(DEVICE).eval() state = torch.load(CHECKPOINT, map_location="cpu", weights_only=True)["model"] model.load_state_dict(state) print("P2Echo loaded:", sum(p.numel() for p in model.parameters()) / 1e6, "M params") STRUCTURE_COLORS = { "LV": (255, 60, 60), "MYO": (60, 200, 255), "LA": (255, 220, 60), "RV": (80, 130, 255), "RA": (220, 80, 255), } def preprocess(pil_image): """PIL -> normalized [1,3,256,256] tensor, exactly like p2echo.data.EchoDataset.""" img = pil_image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR) arr = np.array(img).transpose(2, 0, 1) tensor = torch.from_numpy(arr).float() / 255.0 return normalize_image(tensor).unsqueeze(0) @torch.no_grad() def _segment_impl(image, view, structures, threshold): """Reference implementation returning (overlay PIL, gallery list) as plain CPU objects.""" view = view.upper() structures = [s for s in structures if s] or ["LV"] requested = [s for s in structures if s in VIEW_STRUCTURES[view]] if not requested: raise gr.Error(f"None of {structures} is visualized in the {view} view. " f"Choose one of: {', '.join(VIEW_STRUCTURES[view])}.") prompts = [structure_prompt(view, s) for s in requested] emb = text_encoder(prompts, DEVICE).unsqueeze(0) # [1, N, dim] img_t = preprocess(image).to(DEVICE) with torch.autocast("cuda", dtype=torch.bfloat16): logits = model(img_t, emb)[0].float() # [1, N, H, W] probs = torch.sigmoid(logits)[0].cpu().numpy() # [N, H, W] n = len(requested) base = image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR) overlay = np.array(base, dtype=np.float32) for i, s in enumerate(requested): m = probs[i] >= threshold color = np.array(STRUCTURE_COLORS[s], dtype=np.float32) overlay[m] = overlay[m] * 0.45 + color * 0.55 overlay_img = Image.fromarray(overlay.clip(0, 255).astype(np.uint8)) gallery = [(base, "input"), (overlay_img, "overlay")] for i, s in enumerate(requested): m = probs[i] >= threshold color = np.array(STRUCTURE_COLORS[s], dtype=np.float32) tile = np.full((IMAGE_SIZE, IMAGE_SIZE, 3), 255, dtype=np.float32) tile[m] = color gallery.append((Image.fromarray(tile.clip(0, 255).astype(np.uint8)), f"{s} ({float(probs[i].max()):.2f} max)")) return overlay_img, gallery @spaces.GPU(duration=10) def segment(image, view, structures=("LV",), threshold=0.5): """Segment the selected cardiac structures in an echocardiography frame. Args: image: input echocardiography image (any size; resized to 256x256). view: echocardiographic view, one of 2CH / 3CH / 4CH / PSAX. structures: list of structures to segment (LV, MYO, LA, RV, RA). threshold: probability threshold binarising each structure's mask. Returns: (overlay image, gallery of input/overlay/per-structure colour masks). """ if image is None: raise gr.Error("Please upload an echocardiography image first.") return _segment_impl(image, view, structures, float(threshold)) VIEW_DESCRIPTIONS = { "4CH": "Apical 4-Chamber", "2CH": "Apical 2-Chamber", "3CH": "Apical 3-Chamber", "PSAX": "Parasternal Short-Axis", } def update_structures(view): """Restrict the structure checkboxes to those visible in the chosen view.""" return gr.update(choices=list(VIEW_STRUCTURES[view.upper()]), value=["LV"] if "LV" in VIEW_STRUCTURES[view.upper()] else list(VIEW_STRUCTURES[view.upper()])[:1]) CSS = """ #col-container { max-width: 1210px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(title="P2Echo · Text-Prompted Echo Segmentation") as demo: with gr.Column(elem_id="col-container"): gr.Markdown( """ # 🫀 P2Echo · Text-Prompted Echocardiography Segmentation **P2Echo** segments cardiac structures (LV, MYO, LA, RV, RA) across four echo views (2CH, 3CH, 4CH, PSAX) with a single 37.5M-parameter model controlled by a structured text prompt naming the view and the requested structure. Upload a frame, pick the view, select the structures — the demo builds the paper's view-aware prompt (e.g. *"Apical 4-Chamber (4CH): … Segment the LV: The LV appears as a bullet-shaped chamber forming the apex."*) and runs one forward pass. [Paper](https://arxiv.org/abs/2611.13425) · [Code](https://github.com/moeinheidari7829/P2Echo) · [Weights](https://huggingface.co/moein99/P2Echo) — research use only, not for clinical decisions. """ ) with gr.Row(): with gr.Column(scale=5): input_img = gr.Image(type="pil", label="Echocardiography frame", height=320) view = gr.Radio(list(VIEW_DESCRIPTIONS.keys()), value="4CH", label="View", info="Standard echocardiographic view of the frame") structures = gr.CheckboxGroup(list(VIEW_STRUCTURES["4CH"]), value=["LV"], label="Structures to segment", info="Restricted to structures visible in the view") threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Threshold") run_btn = gr.Button("Segment", variant="primary") prompt_preview = gr.Textbox(label="Text prompt sent to the model", lines=4, interactive=False) with gr.Column(scale=7): overlay_out = gr.Image(type="pil", label="Prediction overlay", height=320) gallery_out = gr.Gallery(label="Input · Overlay · Per-structure masks", columns=4, height=300) gr.Examples( examples=[ ["examples/4ch.png", "4CH"], ["examples/2ch.png", "2CH"], ["examples/psax.png", "PSAX"], ], inputs=[input_img, view], outputs=[overlay_out, gallery_out], fn=segment, cache_examples=True, cache_mode="lazy", label="Examples (click one, then Segment)", ) gr.Markdown( """**Reference**: Heidari et al., *Prompt2Echo (P2Echo): Text-Prompted Multi-View Echocardiography Segmentation with One Model*, arXiv:2611.13425. Text encoder: frozen Qwen3-Embedding-0.6B.""" ) view.change(update_structures, inputs=view, outputs=structures) def _preview(view, structures): chosen = [s for s in structures if s in VIEW_STRUCTURES[view.upper()]][:1] if not chosen: return gr.update() return structure_prompt(view.upper(), chosen[0]) structures.change(_preview, inputs=[view, structures], outputs=prompt_preview) view.change(_preview, inputs=[view, structures], outputs=prompt_preview) run_btn.click( segment, inputs=[input_img, view, structures, threshold], outputs=[overlay_out, gallery_out], api_name="segment", ) demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)