Support raw imaging zip uploads

RL0910/enhanced_chat_ui.py

@@ -1051,7 +1051,7 @@ def create_gradio_interface():
 
                         gr.Markdown("### Ovarian Cancer Existing Progression Viewer")
                         gr.Markdown(
-                            "> Existing progression visualization only. This workspace shows preloaded TCGA-OV CT time series or uploaded RLDT imaging packages, with heuristic lesion highlighting when no mask is available."
+                            "> Existing progression visualization only. This workspace shows preloaded TCGA-OV CT time series, uploaded RLDT imaging packages, or uploaded raw CT zip files that will be converted on load."
                         )
 
                         with gr.Row():
@@ -1059,7 +1059,7 @@ def create_gradio_interface():
                                 choices=["Demo Cases", "Upload Package"],
                                 value="Demo Cases",
                                 label="Imaging source",
-                                info="Use the preloaded TCGA-OV cases or upload one packaged case for visualization."
+                                info="Use the preloaded TCGA-OV cases or upload one zip file for visualization. Raw DICOM/NIfTI zip files are converted automatically."
                             )
                             imaging_case_dropdown = gr.Dropdown(
                                 choices=imaging_demo_choices,
@@ -1101,19 +1101,19 @@ def create_gradio_interface():
                         with gr.Row():
                             with gr.Column(scale=2):
                                 imaging_upload = gr.File(
-                                    label="Upload Imaging Package (.zip)",
+                                    label="Upload Imaging ZIP (.zip)",
                                     file_types=[".zip"],
                                     visible=False,
                                 )
                             with gr.Column(scale=1):
                                 imaging_load_btn = _register_button_with_help(
                                     "Load Imaging Case",
-                                    "Load the selected demo case or uploaded imaging package into the progression viewer.",
+                                    "Load the selected demo case or uploaded zip into the progression viewer. Raw source zips may take longer because they are converted on the fly.",
                                     variant="primary",
                                 )
 
                         imaging_requirements = gr.Markdown(imaging_upload_requirements_md())
-                        imaging_case_info = gr.Markdown("Choose a demo case or upload a package, then click **Load Imaging Case**.")
+                        imaging_case_info = gr.Markdown("Choose a demo case or upload a zip file, then click **Load Imaging Case**.")
                         imaging_timepoint_slider = gr.Slider(
                             minimum=0,
                             maximum=0,

RL0910/imaging_viewer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import hashlib
 import io
 import json
 import math
@@ -23,6 +24,8 @@ MODULE_DIR = Path(__file__).resolve().parent
 DEMO_ROOT = MODULE_DIR / "demo_imaging_cases"
 UPLOAD_CACHE_ROOT = Path(tempfile.gettempdir()) / "rldt_ov_imaging_uploads"
 PACKAGE_MAGIC = "rldt_ov_imaging_package"
+RAW_PACKAGE_MAGIC = "rldt_ov_raw_imaging_source"
+TARGET_SHAPE = (80, 160, 160)
 
 
 def _json_load(path: Path) -> dict[str, Any]:
@@ -466,12 +469,225 @@ def _package_root_from_upload(upload_path: str) -> Path:
     with zipfile.ZipFile(src, "r") as zf:
         zf.extractall(target_dir)
     manifest_path = target_dir / "manifest.json"
-    if not manifest_path.exists():
-        raise ValueError("Uploaded package is missing manifest.json.")
-    manifest = _json_load(manifest_path)
-    if manifest.get("format") != PACKAGE_MAGIC:
-        raise ValueError("Uploaded package is not a supported RLDT ovarian imaging package.")
-    return target_dir
+    if manifest_path.exists():
+        manifest = _json_load(manifest_path)
+        if manifest.get("format") == PACKAGE_MAGIC:
+            return target_dir
+
+    converted_root = target_dir / "_converted_package"
+    converted_manifest = converted_root / "manifest.json"
+    if converted_manifest.exists():
+        manifest = _json_load(converted_manifest)
+        if manifest.get("format") == PACKAGE_MAGIC:
+            return converted_root
+
+    return _convert_raw_upload_to_package(target_dir, converted_root, src.name)
+
+
+def _lazy_import_sitk():
+    import SimpleITK as sitk
+
+    return sitk
+
+
+def _read_series_dir(series_dir: Path) -> tuple[np.ndarray, tuple[float, float, float], str]:
+    sitk = _lazy_import_sitk()
+    reader = sitk.ImageSeriesReader()
+    ids = reader.GetGDCMSeriesIDs(str(series_dir))
+    if not ids:
+        raise FileNotFoundError(f"No readable DICOM series found in {series_dir}")
+    files = reader.GetGDCMSeriesFileNames(str(series_dir), ids[0])
+    reader.SetFileNames(files)
+    image = reader.Execute()
+    arr = sitk.GetArrayFromImage(image).astype(np.int16)
+    spacing = image.GetSpacing()  # x, y, z
+    description = ""
+    try:
+        if image.HasMetaDataKey("0008|103e"):
+            description = image.GetMetaData("0008|103e")
+    except Exception:
+        description = ""
+    return arr, (float(spacing[2]), float(spacing[1]), float(spacing[0])), description
+
+
+def _read_nifti_file(path: Path) -> tuple[np.ndarray, tuple[float, float, float], str]:
+    sitk = _lazy_import_sitk()
+    image = sitk.ReadImage(str(path))
+    arr = sitk.GetArrayFromImage(image).astype(np.int16)
+    spacing = image.GetSpacing()  # x, y, z
+    return arr, (float(spacing[2]), float(spacing[1]), float(spacing[0])), path.stem
+
+
+def _resample_nn(arr: np.ndarray, out_shape: tuple[int, int, int], order: int) -> np.ndarray:
+    zoom = [o / i for o, i in zip(out_shape, arr.shape)]
+    return ndi.zoom(arr, zoom=zoom, order=order)
+
+
+def _build_body_mask_hu(volume_hu: np.ndarray) -> np.ndarray:
+    body = volume_hu > -350
+    labels, n = ndi.label(body)
+    if n <= 0:
+        return body
+    sizes = np.bincount(labels.ravel())
+    sizes[0] = 0
+    return labels == int(sizes.argmax())
+
+
+def _heuristic_lesion_mask(volume_hu: np.ndarray, body_mask: np.ndarray) -> tuple[np.ndarray, np.ndarray, float, str]:
+    z_idx = np.where(body_mask.any(axis=(1, 2)))[0]
+    if len(z_idx) == 0:
+        center = np.array([volume_hu.shape[0] // 2, volume_hu.shape[1] // 2, volume_hu.shape[2] // 2], dtype=np.int16)
+        lesion = np.zeros_like(body_mask, dtype=np.uint8)
+        lesion[max(center[0] - 3, 0):center[0] + 4, max(center[1] - 10, 0):center[1] + 11, max(center[2] - 10, 0):center[2] + 11] = 1
+        return lesion, center, 0.1, "fallback_box"
+    z0 = int(np.percentile(z_idx, 60))
+    z1 = int(np.percentile(z_idx, 90))
+    proj = body_mask[z0:z1].max(axis=0)
+    yy, xx = np.where(proj)
+    cy = float(np.mean(yy)) if len(yy) else volume_hu.shape[1] / 2
+    cx = float(np.mean(xx)) if len(xx) else volume_hu.shape[2] / 2
+    h, w = volume_hu.shape[1:]
+    y_grid, x_grid = np.ogrid[:h, :w]
+    central = ((y_grid - cy) ** 2 / (0.18 * h) ** 2 + (x_grid - cx) ** 2 / (0.16 * w) ** 2) <= 1.0
+    roi = np.zeros_like(body_mask, dtype=bool)
+    roi[z0:z1] = central
+    candidate = (volume_hu > 145) & (volume_hu < 280) & body_mask & roi
+    candidate = ndi.binary_opening(candidate, structure=np.ones((1, 3, 3)))
+    candidate = ndi.binary_closing(candidate, structure=np.ones((1, 5, 5)))
+    labels, n = ndi.label(candidate)
+    best_mask = None
+    best_score = -1.0
+    center = np.array([int((z0 + z1) / 2), int(cy), int(cx)], dtype=np.int16)
+    if n > 0:
+        coords_center = center.astype(float)
+        for idx in range(1, n + 1):
+            comp = labels == idx
+            vox = int(comp.sum())
+            if vox < 200:
+                continue
+            pts = np.argwhere(comp)
+            centroid = pts.mean(axis=0)
+            dist = float(np.linalg.norm((centroid - coords_center) / np.array([8.0, 18.0, 18.0])))
+            score = vox / (1.0 + dist * 10.0)
+            if score > best_score:
+                best_score = score
+                best_mask = comp
+                center = centroid.astype(np.int16)
+    if best_mask is None:
+        best_mask = np.zeros_like(body_mask, dtype=bool)
+        cz, cyi, cxi = [int(v) for v in center]
+        rz, ry, rx = 4, 12, 12
+        z_grid, y_grid, x_grid = np.ogrid[:volume_hu.shape[0], :volume_hu.shape[1], :volume_hu.shape[2]]
+        ellipsoid = ((z_grid - cz) ** 2 / rz**2 + (y_grid - cyi) ** 2 / ry**2 + (x_grid - cxi) ** 2 / rx**2) <= 1.0
+        best_mask = ellipsoid & body_mask
+        confidence = 0.22
+        source = "heuristic_ellipsoid"
+    else:
+        confidence = 0.42
+        source = "heuristic_component"
+    return best_mask.astype(np.uint8), center.astype(np.int16), confidence, source
+
+
+def _normalize_to_u8(volume_hu: np.ndarray) -> np.ndarray:
+    windowed = np.clip(volume_hu, -150, 250)
+    scaled = ((windowed + 150.0) / 400.0) * 255.0
+    return scaled.astype(np.uint8)
+
+
+def _discover_raw_timepoints(extracted_root: Path) -> list[dict[str, Any]]:
+    nii_files = sorted(
+        [
+            path
+            for path in extracted_root.rglob("*")
+            if path.is_file() and (path.name.endswith(".nii") or path.name.endswith(".nii.gz"))
+        ]
+    )
+    if nii_files:
+        return [
+            {
+                "kind": "nifti",
+                "path": path,
+                "label": path.stem.replace(".nii", ""),
+                "relative_time": float(idx + 1),
+            }
+            for idx, path in enumerate(nii_files)
+        ]
+
+    dicom_dirs: list[Path] = []
+    for candidate in sorted({path.parent for path in extracted_root.rglob("*.dcm")}):
+        if any(child.is_file() and child.suffix.lower() == ".dcm" for child in candidate.iterdir()):
+            dicom_dirs.append(candidate)
+    if dicom_dirs:
+        return [
+            {
+                "kind": "dicom",
+                "path": path,
+                "label": path.name,
+                "relative_time": float(idx + 1),
+            }
+            for idx, path in enumerate(sorted(dicom_dirs))
+        ]
+
+    raise ValueError(
+        "Uploaded zip is neither an RLDT imaging package nor a supported raw CT source zip. "
+        "Expected one case containing either multiple NIfTI volumes or multiple DICOM series folders."
+    )
+
+
+def _convert_raw_upload_to_package(extracted_root: Path, converted_root: Path, upload_name: str) -> Path:
+    converted_root.mkdir(parents=True, exist_ok=True)
+    discovered = _discover_raw_timepoints(extracted_root)
+    case_id = hashlib.sha1(upload_name.encode("utf-8")).hexdigest()[:10]
+    manifest = {
+        "format": PACKAGE_MAGIC,
+        "version": 1,
+        "case_id": f"uploaded-{case_id}",
+        "patient_id": f"uploaded-{case_id}",
+        "display_name": f"Uploaded CT Case {case_id}",
+        "modality": "CT",
+        "summary_note": "Existing progression visualization built from an uploaded raw CT zip. Lesion highlighting is heuristic when no mask is available.",
+        "timepoints": [],
+    }
+
+    for tp_idx, info in enumerate(discovered, start=1):
+        if info["kind"] == "dicom":
+            volume_hu, spacing_zyx, series_description = _read_series_dir(info["path"])
+            series_label = series_description or info["label"]
+        else:
+            volume_hu, spacing_zyx, series_label = _read_nifti_file(info["path"])
+
+        body = _build_body_mask_hu(volume_hu)
+        volume_small = _resample_nn(volume_hu, TARGET_SHAPE, order=1).astype(np.int16)
+        body_small = _resample_nn(body.astype(np.uint8), TARGET_SHAPE, order=0) > 0
+        lesion_mask, roi_center, confidence, source = _heuristic_lesion_mask(volume_small, body_small)
+        volume_u8 = _normalize_to_u8(volume_small)
+
+        asset_name = f"tp_{tp_idx:02d}.json"
+        payload = {
+            "shape": list(volume_u8.shape),
+            "roi_center_zyx": [int(v) for v in roi_center.tolist()],
+            "volume_u8_b85": base64.b85encode(zlib.compress(volume_u8.tobytes(), level=9)).decode("ascii"),
+            "lesion_mask_b85": base64.b85encode(zlib.compress(lesion_mask.astype(np.uint8).tobytes(), level=9)).decode("ascii"),
+        }
+        (converted_root / asset_name).write_text(json.dumps(payload), encoding="utf-8")
+        manifest["timepoints"].append(
+            {
+                "timepoint_id": f"t{tp_idx:02d}",
+                "label": str(info["label"]),
+                "relative_time": float(info["relative_time"]),
+                "asset_path": asset_name,
+                "series_description": str(series_label),
+                "num_slices_original": int(volume_hu.shape[0]),
+                "spacing_zyx_mm": [float(v) for v in spacing_zyx],
+                "lesion_confidence": float(confidence),
+                "lesion_source": source,
+                "lesion_voxel_count": int(lesion_mask.sum()),
+                "source_type": info["kind"],
+            }
+        )
+
+    (converted_root / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
+    return converted_root
 
 
 def available_demo_case_choices() -> list[tuple[str, str]]:
@@ -684,8 +900,9 @@ def select_pointcloud_layer(evt: gr.SelectData, state: dict[str, Any]):
 def imaging_upload_requirements_md() -> str:
     return (
         "#### Imaging package format\n"
-        "- Upload a `.zip` package generated by the RLDT ovarian imaging exporter.\n"
-        "- Each package contains one case, multiple timepoints, and preprocessed `.npz` imaging assets.\n"
+        "- Upload either an RLDT imaging package `.zip` or a raw source `.zip`.\n"
+        "- Raw source zip: one patient, multiple timepoints, each timepoint as a DICOM series folder or a NIfTI volume.\n"
+        "- RLDT package zip: preprocessed internal visualization package with one case and multiple timepoints.\n"
         "- Optional lesion masks are supported. If missing, the viewer falls back to a heuristic pelvic lesion candidate.\n"
         "- This workspace is for **existing progression visualization only**; it does not run predictive modeling."
     )

requirements.txt

@@ -10,6 +10,7 @@ matplotlib>=3.8,<3.10
 plotly>=5.24,<6
 seaborn>=0.13,<0.14
 pillow>=10.2,<11
+SimpleITK>=2.3,<3
 torch>=2.2,<2.6
 tqdm>=4.66,<5
 pyyaml>=6.0,<7