p2echo / app.py
multimodalart's picture
multimodalart HF Staff
Upload app.py with huggingface_hub
85dc9f2 verified
Raw History Blame Contribute Delete
8.4 kB
"""P2Echo: text-prompted multi-view echocardiography segmentation demo.
Upload an echocardiography frame, pick the view and the cardiac structures to
segment, and P2Echo returns a colour-coded segmentation mask following the
structured text prompt built from your selection.
"""
import os
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
import spaces # noqa: E402 (must precede torch)
import torch # noqa: E402
import gradio as gr # noqa: E402
import numpy as np # noqa: E402
from PIL import Image # noqa: E402
from p2echo import P2Echo, TextEncoder # noqa: E402
from p2echo.data import normalize_image # noqa: E402
from p2echo.prompts import VIEW_STRUCTURES, structure_prompt # noqa: E402
DEVICE = "cuda"
IMAGE_SIZE = 256
TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B"
CHECKPOINT = "p2echo.pth" # shipped in the Space repo
# ---------------------------------------------------------------- model load
text_encoder = TextEncoder(TEXT_MODEL).to(DEVICE)
model = P2Echo(text_dim=text_encoder.dim).to(DEVICE).eval()
state = torch.load(CHECKPOINT, map_location="cpu", weights_only=True)["model"]
model.load_state_dict(state)
print("P2Echo loaded:", sum(p.numel() for p in model.parameters()) / 1e6, "M params")
STRUCTURE_COLORS = {
"LV": (255, 60, 60),
"MYO": (60, 200, 255),
"LA": (255, 220, 60),
"RV": (80, 130, 255),
"RA": (220, 80, 255),
}
def preprocess(pil_image):
"""PIL -> normalized [1,3,256,256] tensor, exactly like p2echo.data.EchoDataset."""
img = pil_image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
arr = np.array(img).transpose(2, 0, 1)
tensor = torch.from_numpy(arr).float() / 255.0
return normalize_image(tensor).unsqueeze(0)
@torch.no_grad()
def _segment_impl(image, view, structures, threshold):
"""Reference implementation returning (overlay PIL, gallery list) as plain CPU objects."""
view = view.upper()
structures = [s for s in structures if s] or ["LV"]
requested = [s for s in structures if s in VIEW_STRUCTURES[view]]
if not requested:
raise gr.Error(f"None of {structures} is visualized in the {view} view. "
f"Choose one of: {', '.join(VIEW_STRUCTURES[view])}.")
prompts = [structure_prompt(view, s) for s in requested]
emb = text_encoder(prompts, DEVICE).unsqueeze(0) # [1, N, dim]
img_t = preprocess(image).to(DEVICE)
with torch.autocast("cuda", dtype=torch.bfloat16):
logits = model(img_t, emb)[0].float() # [1, N, H, W]
probs = torch.sigmoid(logits)[0].cpu().numpy() # [N, H, W]
n = len(requested)
base = image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
overlay = np.array(base, dtype=np.float32)
for i, s in enumerate(requested):
m = probs[i] >= threshold
color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
overlay[m] = overlay[m] * 0.45 + color * 0.55
overlay_img = Image.fromarray(overlay.clip(0, 255).astype(np.uint8))
gallery = [(base, "input"), (overlay_img, "overlay")]
for i, s in enumerate(requested):
m = probs[i] >= threshold
color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
tile = np.full((IMAGE_SIZE, IMAGE_SIZE, 3), 255, dtype=np.float32)
tile[m] = color
gallery.append((Image.fromarray(tile.clip(0, 255).astype(np.uint8)), f"{s} ({float(probs[i].max()):.2f} max)"))
return overlay_img, gallery
@spaces.GPU(duration=10)
def segment(image, view, structures=("LV",), threshold=0.5):
"""Segment the selected cardiac structures in an echocardiography frame.
Args:
image: input echocardiography image (any size; resized to 256x256).
view: echocardiographic view, one of 2CH / 3CH / 4CH / PSAX.
structures: list of structures to segment (LV, MYO, LA, RV, RA).
threshold: probability threshold binarising each structure's mask.
Returns:
(overlay image, gallery of input/overlay/per-structure colour masks).
"""
if image is None:
raise gr.Error("Please upload an echocardiography image first.")
return _segment_impl(image, view, structures, float(threshold))
VIEW_DESCRIPTIONS = {
"4CH": "Apical 4-Chamber",
"2CH": "Apical 2-Chamber",
"3CH": "Apical 3-Chamber",
"PSAX": "Parasternal Short-Axis",
}
def update_structures(view):
"""Restrict the structure checkboxes to those visible in the chosen view."""
return gr.update(choices=list(VIEW_STRUCTURES[view.upper()]),
value=["LV"] if "LV" in VIEW_STRUCTURES[view.upper()]
else list(VIEW_STRUCTURES[view.upper()])[:1])
CSS = """
#col-container { max-width: 1210px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""
with gr.Blocks(title="P2Echo · Text-Prompted Echo Segmentation") as demo:
with gr.Column(elem_id="col-container"):
gr.Markdown(
"""
# 🫀 P2Echo · Text-Prompted Echocardiography Segmentation
**P2Echo** segments cardiac structures (LV, MYO, LA, RV, RA) across four echo views
(2CH, 3CH, 4CH, PSAX) with a single 37.5M-parameter model controlled by a structured
text prompt naming the view and the requested structure.
Upload a frame, pick the view, select the structures — the demo builds the paper's
view-aware prompt (e.g. *"Apical 4-Chamber (4CH): … Segment the LV: The LV appears as
a bullet-shaped chamber forming the apex."*) and runs one forward pass.
[Paper](https://arxiv.org/abs/2611.13425) · [Code](https://github.com/moeinheidari7829/P2Echo) ·
[Weights](https://huggingface.co/moein99/P2Echo) — research use only, not for clinical decisions.
"""
)
with gr.Row():
with gr.Column(scale=5):
input_img = gr.Image(type="pil", label="Echocardiography frame", height=320)
view = gr.Radio(list(VIEW_DESCRIPTIONS.keys()), value="4CH", label="View",
info="Standard echocardiographic view of the frame")
structures = gr.CheckboxGroup(list(VIEW_STRUCTURES["4CH"]), value=["LV"],
label="Structures to segment",
info="Restricted to structures visible in the view")
threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Threshold")
run_btn = gr.Button("Segment", variant="primary")
prompt_preview = gr.Textbox(label="Text prompt sent to the model", lines=4, interactive=False)
with gr.Column(scale=7):
overlay_out = gr.Image(type="pil", label="Prediction overlay", height=320)
gallery_out = gr.Gallery(label="Input · Overlay · Per-structure masks",
columns=4, height=300)
gr.Examples(
examples=[
["examples/4ch.png", "4CH"],
["examples/2ch.png", "2CH"],
["examples/psax.png", "PSAX"],
],
inputs=[input_img, view],
outputs=[overlay_out, gallery_out],
fn=segment,
cache_examples=True,
cache_mode="lazy",
label="Examples (click one, then Segment)",
)
gr.Markdown(
"""**Reference**: Heidari et al., *Prompt2Echo (P2Echo): Text-Prompted Multi-View
Echocardiography Segmentation with One Model*, arXiv:2611.13425. Text encoder:
frozen Qwen3-Embedding-0.6B."""
)
view.change(update_structures, inputs=view, outputs=structures)
def _preview(view, structures):
chosen = [s for s in structures if s in VIEW_STRUCTURES[view.upper()]][:1]
if not chosen:
return gr.update()
return structure_prompt(view.upper(), chosen[0])
structures.change(_preview, inputs=[view, structures], outputs=prompt_preview)
view.change(_preview, inputs=[view, structures], outputs=prompt_preview)
run_btn.click(
segment,
inputs=[input_img, view, structures, threshold],
outputs=[overlay_out, gallery_out],
api_name="segment",
)
demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)