File size: 8,399 Bytes
dc80c6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
85dc9f2
dc80c6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3c6c711
dc80c6b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3c6c711
b29d809
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
"""P2Echo: text-prompted multi-view echocardiography segmentation demo.

Upload an echocardiography frame, pick the view and the cardiac structures to
segment, and P2Echo returns a colour-coded segmentation mask following the
structured text prompt built from your selection.
"""

import os

os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import spaces  # noqa: E402  (must precede torch)
import torch  # noqa: E402
import gradio as gr  # noqa: E402
import numpy as np  # noqa: E402
from PIL import Image  # noqa: E402

from p2echo import P2Echo, TextEncoder  # noqa: E402
from p2echo.data import normalize_image  # noqa: E402
from p2echo.prompts import VIEW_STRUCTURES, structure_prompt  # noqa: E402

DEVICE = "cuda"
IMAGE_SIZE = 256
TEXT_MODEL = "Qwen/Qwen3-Embedding-0.6B"

CHECKPOINT = "p2echo.pth"  # shipped in the Space repo

# ---------------------------------------------------------------- model load
text_encoder = TextEncoder(TEXT_MODEL).to(DEVICE)
model = P2Echo(text_dim=text_encoder.dim).to(DEVICE).eval()
state = torch.load(CHECKPOINT, map_location="cpu", weights_only=True)["model"]
model.load_state_dict(state)
print("P2Echo loaded:", sum(p.numel() for p in model.parameters()) / 1e6, "M params")

STRUCTURE_COLORS = {
    "LV": (255, 60, 60),
    "MYO": (60, 200, 255),
    "LA": (255, 220, 60),
    "RV": (80, 130, 255),
    "RA": (220, 80, 255),
}


def preprocess(pil_image):
    """PIL -> normalized [1,3,256,256] tensor, exactly like p2echo.data.EchoDataset."""
    img = pil_image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
    arr = np.array(img).transpose(2, 0, 1)
    tensor = torch.from_numpy(arr).float() / 255.0
    return normalize_image(tensor).unsqueeze(0)


@torch.no_grad()
def _segment_impl(image, view, structures, threshold):
    """Reference implementation returning (overlay PIL, gallery list) as plain CPU objects."""
    view = view.upper()
    structures = [s for s in structures if s] or ["LV"]
    requested = [s for s in structures if s in VIEW_STRUCTURES[view]]
    if not requested:
        raise gr.Error(f"None of {structures} is visualized in the {view} view. "
                      f"Choose one of: {', '.join(VIEW_STRUCTURES[view])}.")
    prompts = [structure_prompt(view, s) for s in requested]
    emb = text_encoder(prompts, DEVICE).unsqueeze(0)            # [1, N, dim]
    img_t = preprocess(image).to(DEVICE)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        logits = model(img_t, emb)[0].float()                   # [1, N, H, W]
    probs = torch.sigmoid(logits)[0].cpu().numpy()              # [N, H, W]

    n = len(requested)
    base = image.convert("RGB").resize((IMAGE_SIZE, IMAGE_SIZE), Image.BILINEAR)
    overlay = np.array(base, dtype=np.float32)
    for i, s in enumerate(requested):
        m = probs[i] >= threshold
        color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
        overlay[m] = overlay[m] * 0.45 + color * 0.55
    overlay_img = Image.fromarray(overlay.clip(0, 255).astype(np.uint8))

    gallery = [(base, "input"), (overlay_img, "overlay")]
    for i, s in enumerate(requested):
        m = probs[i] >= threshold
        color = np.array(STRUCTURE_COLORS[s], dtype=np.float32)
        tile = np.full((IMAGE_SIZE, IMAGE_SIZE, 3), 255, dtype=np.float32)
        tile[m] = color
        gallery.append((Image.fromarray(tile.clip(0, 255).astype(np.uint8)), f"{s} ({float(probs[i].max()):.2f} max)"))
    return overlay_img, gallery


@spaces.GPU(duration=10)
def segment(image, view, structures=("LV",), threshold=0.5):
    """Segment the selected cardiac structures in an echocardiography frame.

    Args:
        image: input echocardiography image (any size; resized to 256x256).
        view: echocardiographic view, one of 2CH / 3CH / 4CH / PSAX.
        structures: list of structures to segment (LV, MYO, LA, RV, RA).
        threshold: probability threshold binarising each structure's mask.

    Returns:
        (overlay image, gallery of input/overlay/per-structure colour masks).
    """
    if image is None:
        raise gr.Error("Please upload an echocardiography image first.")
    return _segment_impl(image, view, structures, float(threshold))


VIEW_DESCRIPTIONS = {
    "4CH": "Apical 4-Chamber",
    "2CH": "Apical 2-Chamber",
    "3CH": "Apical 3-Chamber",
    "PSAX": "Parasternal Short-Axis",
}


def update_structures(view):
    """Restrict the structure checkboxes to those visible in the chosen view."""
    return gr.update(choices=list(VIEW_STRUCTURES[view.upper()]),
                     value=["LV"] if "LV" in VIEW_STRUCTURES[view.upper()]
                     else list(VIEW_STRUCTURES[view.upper()])[:1])


CSS = """
#col-container { max-width: 1210px; margin: 0 auto; }
.dark .gradio-container { color: var(--body-text-color); }
"""

with gr.Blocks(title="P2Echo · Text-Prompted Echo Segmentation") as demo:
    with gr.Column(elem_id="col-container"):
        gr.Markdown(
            """
            # 🫀 P2Echo · Text-Prompted Echocardiography Segmentation

            **P2Echo** segments cardiac structures (LV, MYO, LA, RV, RA) across four echo views
            (2CH, 3CH, 4CH, PSAX) with a single 37.5M-parameter model controlled by a structured
            text prompt naming the view and the requested structure.

            Upload a frame, pick the view, select the structures — the demo builds the paper's
            view-aware prompt (e.g. *"Apical 4-Chamber (4CH): … Segment the LV: The LV appears as
            a bullet-shaped chamber forming the apex."*) and runs one forward pass.

            [Paper](https://arxiv.org/abs/2611.13425) · [Code](https://github.com/moeinheidari7829/P2Echo) ·
            [Weights](https://huggingface.co/moein99/P2Echo) — research use only, not for clinical decisions.
            """
        )
        with gr.Row():
            with gr.Column(scale=5):
                input_img = gr.Image(type="pil", label="Echocardiography frame", height=320)
                view = gr.Radio(list(VIEW_DESCRIPTIONS.keys()), value="4CH", label="View",
                                info="Standard echocardiographic view of the frame")
                structures = gr.CheckboxGroup(list(VIEW_STRUCTURES["4CH"]), value=["LV"],
                                               label="Structures to segment",
                                               info="Restricted to structures visible in the view")
                threshold = gr.Slider(0.1, 0.9, value=0.5, step=0.05, label="Threshold")
                run_btn = gr.Button("Segment", variant="primary")
                prompt_preview = gr.Textbox(label="Text prompt sent to the model", lines=4, interactive=False)
            with gr.Column(scale=7):
                overlay_out = gr.Image(type="pil", label="Prediction overlay", height=320)
                gallery_out = gr.Gallery(label="Input · Overlay · Per-structure masks",
                                         columns=4, height=300)
        gr.Examples(
            examples=[
                ["examples/4ch.png", "4CH"],
                ["examples/2ch.png", "2CH"],
                ["examples/psax.png", "PSAX"],
            ],
            inputs=[input_img, view],
            outputs=[overlay_out, gallery_out],
            fn=segment,
            cache_examples=True,
            cache_mode="lazy",
            label="Examples (click one, then Segment)",
        )
        gr.Markdown(
            """**Reference**: Heidari et al., *Prompt2Echo (P2Echo): Text-Prompted Multi-View
            Echocardiography Segmentation with One Model*, arXiv:2611.13425. Text encoder:
            frozen Qwen3-Embedding-0.6B."""
        )

    view.change(update_structures, inputs=view, outputs=structures)

    def _preview(view, structures):
        chosen = [s for s in structures if s in VIEW_STRUCTURES[view.upper()]][:1]
        if not chosen:
            return gr.update()
        return structure_prompt(view.upper(), chosen[0])

    structures.change(_preview, inputs=[view, structures], outputs=prompt_preview)
    view.change(_preview, inputs=[view, structures], outputs=prompt_preview)

    run_btn.click(
        segment,
        inputs=[input_img, view, structures, threshold],
        outputs=[overlay_out, gallery_out],
        api_name="segment",
    )

demo.launch(theme=gr.themes.Citrus(), css=CSS, mcp_server=True)