Update Streamlit app v9 and self_train v5

self_train.py

@@ -13,6 +13,16 @@ Trigger conditions (any one fires a retrain):
   2. N unlabelled images accumulated in queue          → retrain_queue/
   3. K consecutive low-confidence images              → retrain_queue/ (reason=low_confidence)
   4. Nightly scheduled run                            → APScheduler cron 02:00 UTC
+  5. User-optimized parameters submitted              → corrections/ (reason=user_optimized_params)
+     These submissions include the image, postprocessed masks from the user's
+     tuned parameter set, and a full snapshot of the sidebar settings.  The
+     self-training pipeline uses these as additional supervised training pairs
+     and can aggregate parameter statistics to learn optimal defaults.
+
+Privacy note:
+  Images processed in Private Mode (toggle in the Streamlit sidebar) are
+  NEVER queued for retraining.  Only images explicitly submitted by the
+  user via "Submit for training" or "Submit corrections" are used.
 
 After each retrain:
   • Fine-tunes from current HF Hub weights
@@ -20,6 +30,7 @@ After each retrain:
   • Only pushes to Hub if new Dice > previous best
   • Archives queue → runs/<run_id>/processed_queue/
   • Appends entry to manifest.json
+  • Aggregates user-submitted parameter snapshots → optimal_params.json
 
 Usage:
   python self_train.py              # check triggers once
@@ -118,6 +129,45 @@ def _load_manifest() -> list:
 def _save_manifest(m: list): MANIFEST_PATH.write_text(json.dumps(m, indent=2, default=str))
 
 
+def _aggregate_user_params(corrections_dir: Path, run_dir: Path):
+    """
+    Scan corrections for user_optimized_params submissions and aggregate
+    their parameter settings.  Writes optimal_params.json to run_dir with
+    median values — useful for tuning defaults.
+    """
+    all_params = []
+    if not corrections_dir.exists():
+        return
+    for meta_p in corrections_dir.glob("*/meta.json"):
+        try:
+            meta = json.loads(meta_p.read_text())
+            if meta.get("reason") == "user_optimized_params" and "parameters" in meta:
+                all_params.append(meta["parameters"])
+        except Exception:
+            continue
+
+    if not all_params:
+        return
+
+    # Compute median for each numeric parameter
+    aggregated = {}
+    for key in all_params[0]:
+        vals = [p[key] for p in all_params if key in p and isinstance(p[key], (int, float))]
+        if vals:
+            vals.sort()
+            mid = len(vals) // 2
+            aggregated[key] = vals[mid] if len(vals) % 2 else (vals[mid-1] + vals[mid]) / 2
+
+    result = {
+        "n_submissions": len(all_params),
+        "aggregated_params": aggregated,
+        "all_submissions": all_params,
+    }
+    out = run_dir / "optimal_params.json"
+    out.write_text(json.dumps(result, indent=2))
+    log.info("Aggregated %d user param submissions → %s", len(all_params), out)
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Trigger checks
 # ─────────────────────────────────────────────────────────────────────────────
@@ -431,6 +481,10 @@ def run_retrain(reason: str = "scheduled"):
     # Archive queue
     archive = run_dir / "processed_queue"
     archive.mkdir(parents=True, exist_ok=True)
+
+    # Before archiving, collect user-submitted parameter snapshots
+    _aggregate_user_params(CORRECTIONS_DIR, run_dir)
+
     for p in list(QUEUE_DIR.glob("*")) if QUEUE_DIR.exists() else []:
         shutil.move(str(p), str(archive / p.name))
     for folder in list(CORRECTIONS_DIR.glob("*")) if CORRECTIONS_DIR.exists() else []:

src/streamlit_app.py

@@ -7,7 +7,7 @@ Drop-in replacement for streamlit_app.py on Hugging Face Spaces.
 Features:
   ✦ Animated count-up metrics (9 counters)
   ✦ Instance overlay — nucleus IDs (1,2,3…) + myotube IDs (M1,M2…)
-  ✦ Contour outline overlay — see exactly what each detection covers
+  ✦ Separate nuclei outline + myotube outline tabs
   ✦ Watershed nuclei splitting for accurate counts
   ✦ Myotube surface area (total, mean, max µm²) + per-tube bar chart
   ✦ Active learning — upload corrected masks → saved to corrections/
@@ -15,14 +15,24 @@ Features:
   ✦ Retraining queue status panel
   ✦ All original sidebar controls preserved
 
-  v8 changes (validated against 57-well manual count dataset):
-  ✦ REMOVED myotube closing — was merging adjacent myotubes into single blobs
-    (caused 86% of images to undercount; r=0.245 → see validation report).
-  ✦ Unified postprocessing: opening + erode/dilate (matches training script).
-  ✦ Added aspect-ratio shape filter to reject round debris false positives.
-  ✦ Added contour outline tab per collaborator request.
-  ✦ Fixed active learning correction upload bug (art["original"] → art["rgb_u8"]).
-  ✦ Unified threshold defaults across all scripts (thr_myo=0.40, thr_nuc=0.45).
+  v9 changes:
+  ✦ FIXED: SVG viewer myotube ID count now matches live metrics count.
+    Root cause: viewer showed all connected components (myo_lab), but
+    metrics only counted those with ≥1 MyHC+ nucleus.  Now the viewer
+    badge shows the biological myotube_count from compute_bio_metrics,
+    and non-bio myotube regions are shown as faint outlines (not labelled).
+  ✦ Outlines split into two separate tabs: "Nuclei outlines" and
+    "Myotube outlines" per collaborator request.
+  ✦ Privacy mode: sidebar toggle for "Private mode — do not use my data
+    for training".  When enabled, images are NOT queued for retraining
+    (no low_confidence queue, no corrections submission).
+  ✦ Training contribution mode: explicit user-initiated action to submit
+    current image + tuned parameters as a training contribution.  Only
+    runs when user clicks "Submit for training" after finding good params.
+  ✦ Parameter learning: when user submits, the current sidebar parameter
+    set (thresholds, postprocessing knobs) is saved alongside the image
+    so self_train.py can learn optimal parameters per image type.
+  ✦ All v8 fixes preserved (no closing, shape filter, erode+dilate, etc).
 """
 
 import io
@@ -480,6 +490,8 @@ def compute_bio_metrics(nuc_mask, myo_mask,
         "mean_area_um2"            : sa["mean_area_um2"],
         "max_area_um2"             : sa["max_area_um2"],
         "_per_myotube_areas"       : sa["per_myotube_areas"],
+        "_bio_myo_ids"             : set(nm.keys()),   # myotube label IDs with ≥1 MyHC+ nucleus
+        "_total_cc_count"          : int(myo_lab.max()),  # total connected components (for reference)
     }
 
 
@@ -596,13 +608,20 @@ def make_outline_overlay(rgb_u8: np.ndarray,
 
 def collect_label_positions(nuc_lab: np.ndarray,
                              myo_lab: np.ndarray,
-                             img_w: int, img_h: int) -> dict:
+                             img_w: int, img_h: int,
+                             bio_myo_ids: set = None) -> dict:
     """
     Collect centroid positions for every nucleus and myotube,
     scaled to the original image pixel dimensions.
+
+    bio_myo_ids: set of myotube label IDs that have ≥1 MyHC+ nucleus.
+                 If provided, only these are labelled as "M1", "M2", …
+                 (renumbered sequentially).  Non-bio regions get no label
+                 and are stored separately for faint outline rendering.
     Returns:
-        { "nuclei":   [ {"id": 1,  "x": 123.4, "y": 56.7}, ... ],
-          "myotubes": [ {"id": "M1","x": 200.1, "y": 300.5}, ... ] }
+        { "nuclei":     [ {"id": "1",  "x": 123.4, "y": 56.7}, ... ],
+          "myotubes":   [ {"id": "M1", "x": 200.1, "y": 300.5, "orig_label": 5}, ... ],
+          "myotubes_nonbio": [ {"id": "", "x": ..., "y": ..., "orig_label": 3}, ... ] }
     """
     sx = img_w / nuc_lab.shape[1]
     sy = img_h / nuc_lab.shape[0]
@@ -615,11 +634,29 @@ def collect_label_positions(nuc_lab: np.ndarray,
     sx2 = img_w / myo_lab.shape[1]
     sy2 = img_h / myo_lab.shape[0]
     myotubes = []
-    for prop in measure.regionprops(myo_lab):
-        r, c = prop.centroid
-        myotubes.append({"id": f"M{prop.label}", "x": round(c * sx2, 1), "y": round(r * sy2, 1)})
+    myotubes_nonbio = []
+
+    if bio_myo_ids is not None and len(bio_myo_ids) > 0:
+        # Renumber biological myotubes sequentially: M1, M2, M3…
+        sorted_bio = sorted(bio_myo_ids)
+        bio_remap = {orig: idx + 1 for idx, orig in enumerate(sorted_bio)}
+        for prop in measure.regionprops(myo_lab):
+            r, c = prop.centroid
+            pos = {"x": round(c * sx2, 1), "y": round(r * sy2, 1), "orig_label": prop.label}
+            if prop.label in bio_remap:
+                pos["id"] = f"M{bio_remap[prop.label]}"
+                myotubes.append(pos)
+            else:
+                pos["id"] = ""
+                myotubes_nonbio.append(pos)
+    else:
+        # Fallback: label all connected components (backward compat)
+        for prop in measure.regionprops(myo_lab):
+            r, c = prop.centroid
+            myotubes.append({"id": f"M{prop.label}", "x": round(c * sx2, 1),
+                             "y": round(r * sy2, 1), "orig_label": prop.label})
 
-    return {"nuclei": nuclei, "myotubes": myotubes}
+    return {"nuclei": nuclei, "myotubes": myotubes, "myotubes_nonbio": myotubes_nonbio}
 
 
 def make_svg_viewer(img_b64: str,
@@ -1238,6 +1275,23 @@ with st.sidebar:
     st.header("Active learning")
     enable_al = st.toggle("Enable correction upload", value=True)
 
+    st.header("Privacy & Training")
+    private_mode = st.toggle(
+        "🔒 Private mode",
+        value=False,
+        help=(
+            "When enabled, your images are processed locally only. "
+            "They are NOT added to the retraining queue, NOT saved to "
+            "corrections/, and NOT used for model improvement in any way. "
+            "Use this for unpublished data or sensitive research images."
+        )
+    )
+    if private_mode:
+        st.info(
+            "🔒 **Private mode ON** — your images will not be used for "
+            "training or stored beyond this session."
+        )
+
     st.header("Metric definitions")
     with st.expander("Fusion Index"):
         st.write("100 × (nuclei in myotubes with ≥2 nuclei) / total nuclei")
@@ -1250,8 +1304,13 @@ with st.sidebar:
 # ─────────────────────────────────────────────────────────────────────────────
 # FILE UPLOADER
 # ─────────────────────────────────────────────────────────────────────────────
+_uploader_label = (
+    "Upload 1+ images (png / jpg / tif).  🔒 Private mode is ON — images will not be stored."
+    if private_mode else
+    "Upload 1+ images (png / jpg / tif).  Images may be used for model improvement."
+)
 uploads = st.file_uploader(
-    "Upload 1+ images (png / jpg / tif).  Public Space — don't upload sensitive data.",
+    _uploader_label,
     type=["png", "jpg", "jpeg", "tif", "tiff"],
     accept_multiple_files=True,
 )
@@ -1308,12 +1367,13 @@ if run:
                 with torch.no_grad():
                     probs = torch.sigmoid(model(x_t)).cpu().numpy()[0]
 
-                # Confidence check
+                # Confidence check — only queue for training if NOT in private mode
                 conf = float(np.mean([probs[0].max(), probs[1].max()]))
                 if conf < CONF_FLAG_THR:
                     low_conf_flags.append((name, conf))
-                    add_to_queue(rgb_u8, reason="low_confidence",
-                                 metadata={"confidence": conf, "filename": up.name})
+                    if not private_mode:
+                        add_to_queue(rgb_u8, reason="low_confidence",
+                                     metadata={"confidence": conf, "filename": up.name})
 
                 nuc_raw = (probs[0] > float(thr_nuc)).astype(np.uint8)
                 myo_raw = (probs[1] > float(thr_myo)).astype(np.uint8)
@@ -1355,7 +1415,8 @@ if run:
 
                 # Label positions in image-pixel coordinates (used by SVG viewer)
                 orig_h_img, orig_w_img = rgb_u8.shape[:2]
-                label_positions = collect_label_positions(nuc_lab, myo_lab, orig_w_img, orig_h_img)
+                label_positions = collect_label_positions(nuc_lab, myo_lab, orig_w_img, orig_h_img,
+                                                         bio_myo_ids=bio_myo_ids)
 
                 bio = compute_bio_metrics(
                     nuc_pp, myo_pp,
@@ -1368,6 +1429,8 @@ if run:
                 )
                 bio["fi_method"] = "cytoplasm-hole" if use_hole_method else "pixel-overlap"
                 per_areas = bio.pop("_per_myotube_areas", [])
+                bio_myo_ids = bio.pop("_bio_myo_ids", set())
+                total_cc_count = bio.pop("_total_cc_count", 0)
                 bio["image"] = name
                 results.append(bio)
                 all_bio_metrics[name] = {**bio, "_per_myotube_areas": per_areas}
@@ -1503,7 +1566,8 @@ col_img, col_metrics = st.columns([3, 2], gap="large")
 with col_img:
     tabs = st.tabs([
         "🔵 Combined",
-        "📐 Outlines",
+        "📐 Nuclei outlines",
+        "📐 Myotube outlines",
         "🟣 Nuclei only",
         "🟠 Myotubes only",
         "📷 Original",
@@ -1547,41 +1611,58 @@ with col_img:
         st.components.v1.html(html_combined, height=680, scrolling=False)
 
     with tabs[1]:
-        # Outline overlay — shows contour boundaries around each detection
-        outline_img = make_outline_overlay(
-            _rgb, _nl, _ml,
+        # Nuclei-only outlines
+        nuc_outline_img = make_outline_overlay(
+            _rgb, _nl, np.zeros_like(_ml),
             nuc_color=nuc_rgb, myo_color=(0, 255, 0),
             line_width=2,
         )
-        outline_b64 = _b64png_disp(outline_img)
-        outline_lpos = lpos  # show both labels on outline view
-        html_outline = make_svg_viewer(
-            outline_b64, iw, ih, outline_lpos,
-            show_nuclei=label_nuc, show_myotubes=label_myo,
+        nuc_outline_b64 = _b64png_disp(nuc_outline_img)
+        nuc_outline_lpos = {"nuclei": lpos["nuclei"], "myotubes": [], "myotubes_nonbio": []}
+        html_nuc_outline = make_svg_viewer(
+            nuc_outline_b64, iw, ih, nuc_outline_lpos,
+            show_nuclei=True, show_myotubes=False,
         )
-        st.components.v1.html(html_outline, height=680, scrolling=False)
+        st.components.v1.html(html_nuc_outline, height=680, scrolling=False)
 
     with tabs[2]:
-        nuc_only_lpos = {"nuclei": lpos["nuclei"], "myotubes": []}
+        # Myotube-only outlines
+        myo_outline_img = make_outline_overlay(
+            _rgb, np.zeros_like(_nl), _ml,
+            nuc_color=nuc_rgb, myo_color=(0, 255, 0),
+            line_width=2,
+        )
+        myo_outline_b64 = _b64png_disp(myo_outline_img)
+        myo_outline_lpos = {"nuclei": [], "myotubes": lpos["myotubes"],
+                            "myotubes_nonbio": lpos.get("myotubes_nonbio", [])}
+        html_myo_outline = make_svg_viewer(
+            myo_outline_b64, iw, ih, myo_outline_lpos,
+            show_nuclei=False, show_myotubes=True,
+        )
+        st.components.v1.html(html_myo_outline, height=680, scrolling=False)
+
+    with tabs[3]:
+        nuc_only_lpos = {"nuclei": lpos["nuclei"], "myotubes": [], "myotubes_nonbio": []}
         html_nuc = make_svg_viewer(
             nuc_only_b64, iw, ih, nuc_only_lpos,
             show_nuclei=True, show_myotubes=False,
         )
         st.components.v1.html(html_nuc, height=680, scrolling=False)
 
-    with tabs[3]:
-        myo_only_lpos = {"nuclei": [], "myotubes": lpos["myotubes"]}
+    with tabs[4]:
+        myo_only_lpos = {"nuclei": [], "myotubes": lpos["myotubes"],
+                         "myotubes_nonbio": lpos.get("myotubes_nonbio", [])}
         html_myo = make_svg_viewer(
             myo_only_b64, iw, ih, myo_only_lpos,
             show_nuclei=False, show_myotubes=True,
         )
         st.components.v1.html(html_myo, height=680, scrolling=False)
 
-    with tabs[4]:
-        st.image(art["rgb_u8"], use_container_width=True)
     with tabs[5]:
-        st.image(art["nuc_pp"],   use_container_width=True)
+        st.image(art["rgb_u8"], use_container_width=True)
     with tabs[6]:
+        st.image(art["nuc_pp"],   use_container_width=True)
+    with tabs[7]:
         st.image(art["myo_pp"],   use_container_width=True)
 
 with col_metrics:
@@ -1633,10 +1714,70 @@ with col_metrics:
 
 st.divider()
 
+# ─────────────────────────────────────────────────────────────────────────────
+# TRAINING CONTRIBUTION — User-initiated parameter + image submission
+# ─────────────────────────────────────────────────────────────────────────────
+if not private_mode and names:
+    st.subheader("📤 Submit image for training")
+    st.caption(
+        "Once you've tuned the sidebar parameters to get the best results for "
+        "this image, click below to submit both the image and your optimized "
+        "parameters as a training contribution.  This helps MyoSight learn "
+        "better settings for similar images."
+    )
+
+    train_pick = st.selectbox("Image to submit", names, key="train_pick")
+
+    if st.button("📤 Submit for training", type="primary"):
+        _ensure_dirs()
+        ts = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+        folder = CORRECTIONS_DIR / f"params_{ts}"
+        folder.mkdir(parents=True, exist_ok=True)
+
+        # Save the original image
+        train_art = st.session_state.artifacts[train_pick]
+        Image.fromarray(train_art["rgb_u8"]).save(folder / "image.png")
+
+        # Save the postprocessed masks (from current parameter settings)
+        nuc_pp_arr = train_art.get("nuc_pp_arr")
+        myo_pp_arr = train_art.get("myo_pp_arr")
+        if nuc_pp_arr is not None:
+            Image.fromarray((nuc_pp_arr > 0).astype(np.uint8) * 255).save(folder / "nuclei_mask.png")
+        if myo_pp_arr is not None:
+            Image.fromarray((myo_pp_arr > 0).astype(np.uint8) * 255).save(folder / "myotube_mask.png")
+
+        # Save the current parameter set — self_train can learn from these
+        param_snapshot = {
+            "reason": "user_optimized_params",
+            "has_masks": nuc_pp_arr is not None and myo_pp_arr is not None,
+            "timestamp": ts,
+            "source_image": train_pick,
+            "parameters": {
+                "thr_nuc": float(thr_nuc),
+                "thr_myo": float(thr_myo),
+                "min_nuc_area": int(min_nuc_area),
+                "min_myo_area": int(min_myo_area),
+                "nuc_close_radius": int(nuc_close_radius),
+                "myo_open_radius": int(myo_open_radius),
+                "myo_erode_radius": int(myo_erode_radius),
+                "min_myo_aspect_ratio": float(min_myo_aspect_ratio),
+                "myo_max_area_px": int(myo_max_area_px),
+                "myo_split_min_seeds": int(myo_split_min_seeds),
+                "image_size": int(image_size),
+            },
+            "metrics": st.session_state.bio_metrics.get(train_pick, {}),
+        }
+        (folder / "meta.json").write_text(json.dumps(param_snapshot, indent=2, default=str))
+
+        st.success(
+            f"✅ **{train_pick}** submitted for training with your optimized parameters. "
+            "The model will incorporate this at the next retraining cycle."
+        )
+
 # ─────────────────────────────────────────────────────────────────────────────
 # ACTIVE LEARNING — CORRECTION UPLOAD
 # ─────────────────────────────────────────────────────────────────────────────
-if enable_al:
+if enable_al and not private_mode:
     st.subheader("🧠 Submit corrected labels (Active Learning)")
     st.caption(
         "Upload corrected binary masks for any image. "
@@ -1668,6 +1809,11 @@ if enable_al:
                 f"✅ Corrections for **{al_pick}** saved to `corrections/`. "
                 "The model will retrain at the next scheduled cycle."
             )
+elif enable_al and private_mode:
+    st.info(
+        "🔒 Active learning and training submissions are disabled in Private mode. "
+        "Toggle off Private mode in the sidebar to enable."
+    )
 
 st.divider()