Update app.py

app.py

@@ -234,6 +234,44 @@ def _resolve_path(file_obj):
     return ""
 
 
+def _status(points: list) -> str:
+    """Small feedback line so it's obvious when fixations register."""
+    n = len(points) if points else 0
+    if not n:
+        return "_No fixations yet — click the image, or load a fixation file below._"
+    return f"**{n}** fixation(s) placed."
+
+
+def gaze_duration_heatmap(points, height, width, img_w, img_h, sigma=None):
+    """Gaussian-splat heatmap of the fixations, each blob weighted by its
+    dwell / duration (the 3rd component of each point), rendered at
+    (height, width). This is the *observed* gaze heatmap — not the model's
+    learned attention.
+
+    points: list of (x_px, y_px, weight) in original-image pixels.
+    """
+    hm = np.zeros((height, width), dtype=np.float32)
+    if not points:
+        return hm
+    if sigma is None:
+        sigma = max(height, width) / 22.0
+    sx, sy = width / max(img_w, 1), height / max(img_h, 1)
+    rad = max(int(sigma * 3), 1)
+    for x_px, y_px, wgt in points:
+        cx, cy = int(round(x_px * sx)), int(round(y_px * sy))
+        if not (0 <= cx < width and 0 <= cy < height):
+            continue
+        x0, x1 = max(cx - rad, 0), min(cx + rad + 1, width)
+        y0, y1 = max(cy - rad, 0), min(cy + rad + 1, height)
+        xv, yv = np.meshgrid(np.arange(x0, x1), np.arange(y0, y1))
+        g = np.exp(-((xv - cx) ** 2 + (yv - cy) ** 2) / (2.0 * sigma ** 2))
+        # +0.15 floor so short-dwell fixations still register a little.
+        hm[y0:y1, x0:x1] += g.astype(np.float32) * (0.15 + float(wgt))
+    if hm.max() > 0:
+        hm /= hm.max()
+    return hm
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Event handlers — image
 # ─────────────────────────────────────────────────────────────────────────────
@@ -241,7 +279,7 @@ def _resolve_path(file_obj):
 
 def on_file_upload(file_obj):
     """Load any image or DICOM and switch the panel to fixation-click mode."""
-    _no_change = (None, [], "", gr.update(), gr.update(), gr.update())
+    _no_change = (None, [], "", gr.update(), gr.update(), gr.update(), gr.update())
     if file_obj is None:
         return _no_change
 
@@ -265,6 +303,7 @@ def on_file_upload(file_obj):
         gr.update(visible=False),                                   # upload_zone → hide
         gr.update(value=pil, visible=True, label=_FIXATION_LABEL),  # image_panel → show
         gr.update(visible=True),                                    # delete_btn → show
+        _status([]),                                                # fix_status
     )
 
 
@@ -272,17 +311,17 @@ def on_select(orig_image: Image.Image, points: list, weight: float, evt: gr.Sele
     """Record a fixation click in original-image pixel coords."""
     if orig_image is None:
         gr.Warning("Upload an image first.")
-        return points, gr.update()
+        return points, gr.update(), _status(points)
     x_px, y_px = float(evt.index[0]), float(evt.index[1])
     new_points = points + [(x_px, y_px, float(weight))]
-    return new_points, draw_points(orig_image, new_points)
+    return new_points, draw_points(orig_image, new_points), _status(new_points)
 
 
 def on_clear(orig_image):
     """Remove all fixations but keep the current image."""
     if orig_image is None:
-        return [], gr.update()
-    return [], gr.update(value=orig_image)
+        return [], gr.update(), _status([])
+    return [], gr.update(value=orig_image), _status([])
 
 
 def on_delete():
@@ -294,6 +333,7 @@ def on_delete():
         gr.update(value=None, visible=True),     # upload_zone → show (reset)
         gr.update(value=None, visible=False),    # image_panel → hide
         gr.update(visible=False),                # delete_btn → hide
+        _status([]),                             # fix_status
     )
 
 
@@ -376,13 +416,13 @@ def on_apply_fixfile(fixfile_json, id_col, x_col, y_col, time_col, orig_image, i
     ID column doesn't match the uploaded filename)."""
     if orig_image is None:
         gr.Warning("Load an image first, then apply the fixation file.")
-        return gr.update(), gr.update()
+        return gr.update(), gr.update(), gr.update()
     if not fixfile_json:
         gr.Warning("Upload a fixation file first.")
-        return gr.update(), gr.update()
+        return gr.update(), gr.update(), gr.update()
     if not x_col or not y_col:
         gr.Warning("Pick the X and Y columns first.")
-        return gr.update(), gr.update()
+        return gr.update(), gr.update(), gr.update()
 
     df = pd.read_json(fixfile_json)
 
@@ -402,7 +442,7 @@ def on_apply_fixfile(fixfile_json, id_col, x_col, y_col, time_col, orig_image, i
 
     if sub.empty:
         gr.Warning("No usable fixation rows found.")
-        return gr.update(), gr.update()
+        return gr.update(), gr.update(), gr.update()
 
     w, h = orig_image.size
     x_vals = sub[x_col].astype(float).to_numpy()
@@ -413,13 +453,20 @@ def on_apply_fixfile(fixfile_json, id_col, x_col, y_col, time_col, orig_image, i
         t_raw = sub[time_col].astype(float).to_numpy()
         order = np.argsort(t_raw)  # chronological order
         x_px, y_px, t_raw = x_px[order], y_px[order], t_raw[order]
-        tmin, tmax = float(np.nanmin(t_raw)), float(np.nanmax(t_raw))
-        weight = (t_raw - tmin) / (tmax - tmin + 1e-8)
+        # Per-fixation dwell = gap to the next fixation (last one gets the
+        # median gap); normalised to [0,1] so it weights the duration heatmap.
+        if len(t_raw) > 1:
+            dwell = np.diff(t_raw, append=t_raw[-1] + np.median(np.diff(t_raw)))
+            dwell = np.clip(dwell, 0, None)
+            dmax = float(dwell.max())
+            weight = dwell / dmax if dmax > 0 else np.ones_like(dwell)
+        else:
+            weight = np.ones(1)
     else:
-        weight = np.linspace(0.0, 1.0, len(sub))
+        weight = np.ones(len(sub))
 
     new_points = [(float(xp), float(yp), float(wt)) for xp, yp, wt in zip(x_px, y_px, weight)]
-    return new_points, draw_points(orig_image, new_points)
+    return new_points, draw_points(orig_image, new_points), _status(new_points)
 
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -487,19 +534,20 @@ def run(orig_image: Image.Image, points: list, preset_name: str):
     class_probs = {c: float(p) for c, p in zip(predictor.classes, probs)}
     predicted_class = max(class_probs, key=class_probs.get)
 
-    # Learned gaze-conditioned attention mask, upsampled + overlaid.
+    # Output visual: the *observed* gaze-fixation heatmap, each fixation
+    # weighted by its dwell/duration — overlaid on the image and shown raw.
     img_size = predictor.img_size
-    mask_full = patch_to_image(patch_mask[0].cpu().numpy(), img_size, img_size)
     display_img = np.array(orig_image.resize((img_size, img_size)))
-    overlay = make_overlay(display_img, mask_full)
-    mask_img = heatmap_to_image(mask_full)
+    gaze_hm = gaze_duration_heatmap(points, img_size, img_size, w, h)
+    overlay = make_overlay(display_img, gaze_hm)
+    heatmap_img = heatmap_to_image(gaze_hm)
 
     prob_lines = "\n".join(
         f"- **{c}**: {p:.3f}" for c, p in sorted(class_probs.items(), key=lambda kv: -kv[1])
     )
     summary = f"### Predicted: **{predicted_class}**\n\n{prob_lines}"
 
-    return class_probs, summary, overlay, mask_img
+    return class_probs, summary, overlay, heatmap_img
 
 
 # ─────────────────────────────────────────────────────────────────────────────
@@ -540,13 +588,20 @@ with gr.Blocks(title="GazeAlign", css=_CSS) as demo:
                 file_types=[".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".webp", ".dcm"],
                 type="filepath",
             )
+            # interactive=False → a pure display surface that reports click
+            # coordinates via .select (an interactive Image opens an editor
+            # instead and never fires reliable pixel coords).
             image_panel = gr.Image(
-                label=_FIXATION_LABEL, type="pil", interactive=True, visible=False
+                label=_FIXATION_LABEL, type="pil", interactive=False,
+                visible=False, height=440,
+            )
+            fix_status = gr.Markdown(
+                "_No fixations yet — click the image, or load a fixation file below._"
             )
 
             weight_slider = gr.Slider(
                 0.0, 1.0, value=0.5, step=0.05,
-                label="Fixation weight (dwell) for clicks",
+                label="Dwell / duration weight for the next click",
             )
             with gr.Row():
                 clear_btn = gr.Button("Clear fixations")
@@ -568,8 +623,8 @@ with gr.Blocks(title="GazeAlign", css=_CSS) as demo:
             run_btn = gr.Button("Run GazeAlign", variant="primary", elem_id="run-btn")
             label_output = gr.Label(label="Predicted class (probabilities)", num_top_classes=5)
             summary_output = gr.Markdown()
-            overlay_output = gr.Image(label="Gaze-conditioned attention overlay")
-            mask_output = gr.Image(label="Learned attention mask")
+            overlay_output = gr.Image(label="Gaze-fixation heatmap (dwell-weighted) — overlay")
+            mask_output = gr.Image(label="Gaze-fixation heatmap (weighted by duration)")
 
     gr.Markdown(
         "<div class='footer-note'>Fixation tables use raw pixel or normalised "
@@ -582,18 +637,18 @@ with gr.Blocks(title="GazeAlign", css=_CSS) as demo:
     upload_zone.upload(
         on_file_upload,
         [upload_zone],
-        [orig_image_state, points_state, image_name_state, upload_zone, image_panel, delete_btn],
+        [orig_image_state, points_state, image_name_state, upload_zone, image_panel, delete_btn, fix_status],
     )
     image_panel.select(
         on_select,
         [orig_image_state, points_state, weight_slider],
-        [points_state, image_panel],
+        [points_state, image_panel, fix_status],
     )
-    clear_btn.click(on_clear, [orig_image_state], [points_state, image_panel])
+    clear_btn.click(on_clear, [orig_image_state], [points_state, image_panel, fix_status])
     delete_btn.click(
         on_delete,
         None,
-        [orig_image_state, points_state, image_name_state, upload_zone, image_panel, delete_btn],
+        [orig_image_state, points_state, image_name_state, upload_zone, image_panel, delete_btn, fix_status],
     )
 
     fixfile.upload(
@@ -604,7 +659,7 @@ with gr.Blocks(title="GazeAlign", css=_CSS) as demo:
     apply_fix_btn.click(
         on_apply_fixfile,
         [fixfile_df_state, id_col_dd, x_col_dd, y_col_dd, time_col_dd, orig_image_state, image_name_state],
-        [points_state, image_panel],
+        [points_state, image_panel, fix_status],
     )
 
     run_btn.click(