Spaces:
Running on Zero
Running on Zero
File size: 8,399 Bytes
dc80c6b 85dc9f2 dc80c6b 3c6c711 dc80c6b 3c6c711 b29d809 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | """P2Echo: text-prompted multi-view echocardiography segmentation demo.
Upload an echocardiography frame, pick the view and the cardiac structures to
segment, and P2Echo returns a colour-coded segmentation mask following the
structured text prompt built from your selection.
"""
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede torch)
import torch # noqa: E402
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
from PIL import Image # noqa: E402
from p2echo import P2Echo, TextEncoder # noqa: E402
from p2echo.data import normalize_image # noqa: E402
from p2echo.prompts import VIEW_STRUCTURES, structure_prompt # noqa: E402
DEVICE = "cuda"
IMAGE_SIZE = 256
TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B"
CHECKPOINT = "p2echo.pth" # shipped in the Space repo
# ---------------------------------------------------------------- model load
text_encoder = TextEncoder(TEXT_MODEL).to(DEVICE)
model = P2Echo(text_dim=text_encoder.dim).to(DEVICE).eval()
state = torch.load(CHECKPOINT, map_location="cpu", weights_only=True)["model"]
model.load_state_dict(state)
print("P2Echo loaded:", sum(p.numel() for p in model.parameters()) / 1e6, "M params")
STRUCTURE_COLORS = {
"LV": (255, 60, 60),
"MYO": (60, 200, 255),
"LA": (255, 220, 60),
"RV": (80, 130, 255),
"RA": (220, 80, 255),
}
def preprocess(pil_image):
"""PIL -> normalized [1,3,256,256] tensor, exactly like p2echo.data.EchoDataset."""
img = pil_image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
arr = np.array(img).transpose(2, 0, 1)
tensor = torch.from_numpy(arr).float() / 255.0
return normalize_image(tensor).unsqueeze(0)
@torch.no_grad()
def _segment_impl(image, view, structures, threshold):
"""Reference implementation returning (overlay PIL, gallery list) as plain CPU objects."""
view = view.upper()
structures = [s for s in structures if s] or ["LV"]
requested = [s for s in structures if s in VIEW_STRUCTURES[view]]
if not requested:
raise gr.Error(f"None of {structures} is visualized in the {view} view. "
f"Choose one of: {', '.join(VIEW_STRUCTURES[view])}.")
prompts = [structure_prompt(view, s) for s in requested]
emb = text_encoder(prompts, DEVICE).unsqueeze(0) # [1, N, dim]
img_t = preprocess(image).to(DEVICE)
with torch.autocast("cuda", dtype=torch.bfloat16):
logits = model(img_t, emb)[0].float() # [1, N, H, W]
probs = torch.sigmoid(logits)[0].cpu().numpy() # [N, H, W]
n = len(requested)
base = image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
overlay = np.array(base, dtype=np.float32)
for i, s in enumerate(requested):
m = probs[i] >= threshold
color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
overlay[m] = overlay[m] * 0.45 + color * 0.55
overlay_img = Image.fromarray(overlay.clip(0, 255).astype(np.uint8))
gallery = [(base, "input"), (overlay_img, "overlay")]
for i, s in enumerate(requested):
m = probs[i] >= threshold
color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
tile = np.full((IMAGE_SIZE, IMAGE_SIZE, 3), 255, dtype=np.float32)
tile[m] = color
gallery.append((Image.fromarray(tile.clip(0, 255).astype(np.uint8)), f"{s} ({float(probs[i].max()):.2f} max)"))
return overlay_img, gallery
@spaces.GPU(duration=10)
def segment(image, view, structures=("LV",), threshold=0.5):
"""Segment the selected cardiac structures in an echocardiography frame.
Args:
image: input echocardiography image (any size; resized to 256x256).
view: echocardiographic view, one of 2CH / 3CH / 4CH / PSAX.
structures: list of structures to segment (LV, MYO, LA, RV, RA).
threshold: probability threshold binarising each structure's mask.
Returns:
(overlay image, gallery of input/overlay/per-structure colour masks).
"""
if image is None:
raise gr.Error("Please upload an echocardiography image first.")
return _segment_impl(image, view, structures, float(threshold))
VIEW_DESCRIPTIONS = {
"4CH": "Apical 4-Chamber",
"2CH": "Apical 2-Chamber",
"3CH": "Apical 3-Chamber",
"PSAX": "Parasternal Short-Axis",
}
def update_structures(view):
"""Restrict the structure checkboxes to those visible in the chosen view."""
return gr.update(choices=list(VIEW_STRUCTURES[view.upper()]),
value=["LV"] if "LV" in VIEW_STRUCTURES[view.upper()]
else list(VIEW_STRUCTURES[view.upper()])[:1])
CSS = """
#col-container { max-width: 1210px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="P2Echo · Text-Prompted Echo Segmentation") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""
# 🫀 P2Echo · Text-Prompted Echocardiography Segmentation
**P2Echo** segments cardiac structures (LV, MYO, LA, RV, RA) across four echo views
(2CH, 3CH, 4CH, PSAX) with a single 37.5M-parameter model controlled by a structured
text prompt naming the view and the requested structure.
Upload a frame, pick the view, select the structures — the demo builds the paper's
view-aware prompt (e.g. *"Apical 4-Chamber (4CH): … Segment the LV: The LV appears as
a bullet-shaped chamber forming the apex."*) and runs one forward pass.
[Paper](https://arxiv.org/abs/2611.13425) · [Code](https://github.com/moeinheidari7829/P2Echo) ·
[Weights](https://huggingface.co/moein99/P2Echo) — research use only, not for clinical decisions.
"""
)
with gr.Row():
with gr.Column(scale=5):
input_img = gr.Image(type="pil", label="Echocardiography frame", height=320)
view = gr.Radio(list(VIEW_DESCRIPTIONS.keys()), value="4CH", label="View",
info="Standard echocardiographic view of the frame")
structures = gr.CheckboxGroup(list(VIEW_STRUCTURES["4CH"]), value=["LV"],
label="Structures to segment",
info="Restricted to structures visible in the view")
threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Threshold")
run_btn = gr.Button("Segment", variant="primary")
prompt_preview = gr.Textbox(label="Text prompt sent to the model", lines=4, interactive=False)
with gr.Column(scale=7):
overlay_out = gr.Image(type="pil", label="Prediction overlay", height=320)
gallery_out = gr.Gallery(label="Input · Overlay · Per-structure masks",
columns=4, height=300)
gr.Examples(
examples=[
["examples/4ch.png", "4CH"],
["examples/2ch.png", "2CH"],
["examples/psax.png", "PSAX"],
],
inputs=[input_img, view],
outputs=[overlay_out, gallery_out],
fn=segment,
cache_examples=True,
cache_mode="lazy",
label="Examples (click one, then Segment)",
)
gr.Markdown(
"""**Reference**: Heidari et al., *Prompt2Echo (P2Echo): Text-Prompted Multi-View
Echocardiography Segmentation with One Model*, arXiv:2611.13425. Text encoder:
frozen Qwen3-Embedding-0.6B."""
)
view.change(update_structures, inputs=view, outputs=structures)
def _preview(view, structures):
chosen = [s for s in structures if s in VIEW_STRUCTURES[view.upper()]][:1]
if not chosen:
return gr.update()
return structure_prompt(view.upper(), chosen[0])
structures.change(_preview, inputs=[view, structures], outputs=prompt_preview)
view.change(_preview, inputs=[view, structures], outputs=prompt_preview)
run_btn.click(
segment,
inputs=[input_img, view, structures, threshold],
outputs=[overlay_out, gallery_out],
api_name="segment",
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)
|