fix(scripts): carry non-image files through resize + tar shards

The IMG_EXTS filter silently dropped reports (.txt), CheXpert labels
(.csv), and any other metadata interleaved in the source tree, so the
shipped MIMIC-CXR_resized would lose them. Add a parallel _copy_one
worker that copies non-image files verbatim, preserving the exact tree.
Manifest now records both counts. Applies to both build_resized_dataset.py
and resize_and_shard.ipynb (cell c06).

This makes the resized release a faithful mirror of MIMIC-CXR_processed
regardless of what's mixed in -- needed if the instruct JSON builder is
ever run on the training box (auto-build path reads .txt + chexpert.csv).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

scripts/build_resized_dataset.py

@@ -123,27 +123,53 @@ def _resize_one(args) -> tuple[str, str]:
         return f"error:{type(e).__name__}: {e}", rel
 
 
+def _copy_one(args) -> tuple[str, str]:
+    """Worker: copy a non-image file verbatim, preserving the tree.
+
+    Used for reports (.txt), CheXpert labels (.csv), metadata (.json) and
+    anything else interleaved in the source tree -- so the tar shards carry
+    a complete copy of MIMIC-CXR_processed, not just images.
+    """
+    src_path, dst_path, rel = args
+    try:
+        dst_path = Path(dst_path)
+        if dst_path.exists() and dst_path.stat().st_size > 0:
+            return "skipped", rel
+        dst_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(src_path, dst_path)
+        return "copied_other", rel
+    except Exception as e:
+        return f"error:{type(e).__name__}: {e}", rel
+
+
 def resize_tree(src: Path, dst: Path, target: int, quality: int,
                 workers: int, square: bool) -> None:
     print(f"[resize] scanning {src} ...")
-    jobs = []
+    img_jobs, other_jobs = [], []
     for root, _, files in os.walk(src):
         for fn in files:
+            sp = Path(root) / fn
+            rel = sp.relative_to(src)
+            dp = dst / rel
             if fn.lower().endswith(IMG_EXTS):
-                sp = Path(root) / fn
-                rel = sp.relative_to(src)
-                jobs.append((str(sp), str(dst / rel), str(rel), target, quality, square))
-    if not jobs:
-        sys.exit(f"ERROR: no images ({IMG_EXTS}) found under {src}")
+                img_jobs.append((str(sp), str(dp), str(rel), target, quality, square))
+            else:
+                # non-image: reports/csv/json/etc. copied verbatim so the
+                # shipped tree mirrors the source exactly (no data loss).
+                other_jobs.append((str(sp), str(dp), str(rel)))
+    if not img_jobs and not other_jobs:
+        sys.exit(f"ERROR: nothing found under {src}")
     mode = f"square {target}x{target}" if square else f"shortest-edge {target}px"
-    print(f"[resize] {len(jobs):,} images -> {dst}  "
-          f"({mode}, q{quality}, {workers} workers)")
+    print(f"[resize] {len(img_jobs):,} images + {len(other_jobs):,} non-image "
+          f"-> {dst}  ({mode}, q{quality}, {workers} workers)")
 
-    counts = {"resized": 0, "squared": 0, "copied": 0, "skipped": 0, "error": 0}
+    counts = {"resized": 0, "squared": 0, "copied": 0,
+              "copied_other": 0, "skipped": 0, "error": 0}
     errors: list[str] = []
     with ProcessPoolExecutor(max_workers=workers) as ex:
-        futs = [ex.submit(_resize_one, j) for j in jobs]
-        for f in tqdm(as_completed(futs), total=len(futs), unit="img"):
+        futs = [ex.submit(_resize_one, j) for j in img_jobs]
+        futs += [ex.submit(_copy_one, j) for j in other_jobs]
+        for f in tqdm(as_completed(futs), total=len(futs), unit="file"):
             status, rel = f.result()
             if status.startswith("error:"):
                 counts["error"] += 1
@@ -152,13 +178,15 @@ def resize_tree(src: Path, dst: Path, target: int, quality: int,
                 counts[status] += 1
 
     dst.mkdir(parents=True, exist_ok=True)
+    total = len(img_jobs) + len(other_jobs)
     out_bytes = sum(p.stat().st_size for p in dst.rglob("*") if p.is_file())
     (dst / "_manifest.json").write_text(json.dumps({
         "source": str(src), "target": target,
         "mode": "square" if square else "shortest_edge",
         "jpeg_quality": quality, "subsampling": "4:4:4",
         "resampling": "BICUBIC" if square else "LANCZOS",
-        "counts": counts, "total": len(jobs),
+        "counts": counts, "total": total,
+        "images": len(img_jobs), "non_image": len(other_jobs),
         "output_bytes": out_bytes,
         "built_at": time.strftime("%Y-%m-%dT%H:%M:%S"),
     }, indent=2), encoding="utf-8")
@@ -167,7 +195,7 @@ def resize_tree(src: Path, dst: Path, target: int, quality: int,
         print(f"[resize] WARNING: {len(errors)} failures -> {dst/'_errors.txt'}")
     print(f"[resize] done: {counts}")
     print(f"[resize] output size: {out_bytes / 1024**3:.2f} GB "
-          f"({out_bytes / max(1, len(jobs)) / 1024:.0f} KB/image avg)")
+          f"({out_bytes / max(1, len(img_jobs)) / 1024:.0f} KB/image avg)")
 
 
 # -- Phase 2: pack into tar shards -------------------------------------------

scripts/resize_and_shard.ipynb

@@ -46,7 +46,7 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": "import os, json, shutil, tarfile, time\nfrom pathlib import Path\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nImage.MAX_IMAGE_PIXELS = None          # don't abort on large medical images\nIMG_EXTS = (\".jpg\", \".jpeg\", \".png\")\n\n\ndef _resize_one(src_path, dst_path, target, quality, square):\n    \"\"\"Returns one of: resized | squared | copied | skipped | error:<msg>.\"\"\"\n    try:\n        dst_path = Path(dst_path)\n        if dst_path.exists() and dst_path.stat().st_size > 0:\n            return \"skipped\"                       # resumable\n        dst_path.parent.mkdir(parents=True, exist_ok=True)\n        with Image.open(src_path) as im:\n            w, h = im.size\n            shorter = min(w, h)\n            # Non-square: if shorter side already <= target, downscaling would\n            # push it below 518 -> copy verbatim (lossless, never worsens a\n            # low-res source). Square mode must always emit exactly target^2.\n            if not square and shorter <= target:\n                shutil.copy2(src_path, dst_path)\n                return \"copied\"\n            if im.mode not in (\"L\", \"RGB\"):\n                im = im.convert(\"RGB\")\n            # shorter axis EXACTLY = target; longer scales proportionally\n            if w <= h:\n                new_size = (target, round(h * target / w))\n            else:\n                new_size = (round(w * target / h), target)\n            # square mode reproduces the processor exactly -> bicubic\n            im = im.resize(new_size, Image.BICUBIC if square else Image.LANCZOS)\n            if square:\n                W, H = im.size\n                left, top = (W - target) // 2, (H - target) // 2\n                im = im.crop((left, top, left + target, top + target))\n            im.save(dst_path, \"JPEG\", quality=quality, optimize=True, subsampling=0)\n        return \"squared\" if square else \"resized\"\n    except Exception as e:\n        return f\"error:{type(e).__name__}: {e}\"\n\n\ndef resize_tree(src: Path, dst: Path, target, quality, workers, square):\n    print(f\"[resize] scanning {src} ...\")\n    jobs = []\n    for root, _, files in os.walk(src):\n        for fn in files:\n            if fn.lower().endswith(IMG_EXTS):\n                sp = Path(root) / fn\n                rel = sp.relative_to(src)\n                jobs.append((str(sp), str(dst / rel)))\n    if not jobs:\n        raise SystemExit(f\"ERROR: no images under {src}\")\n    mode = f\"square {target}x{target}\" if square else f\"shortest-edge {target}px\"\n    print(f\"[resize] {len(jobs):,} images -> {dst}  ({mode}, q{quality}, {workers} threads)\")\n\n    counts = {\"resized\": 0, \"squared\": 0, \"copied\": 0, \"skipped\": 0, \"error\": 0}\n    errors = []\n    with ThreadPoolExecutor(max_workers=workers) as ex:\n        futs = {ex.submit(_resize_one, s, d, target, quality, square): d\n                for s, d in jobs}\n        for f in tqdm(as_completed(futs), total=len(futs), unit=\"img\"):\n            st = f.result()\n            if st.startswith(\"error:\"):\n                counts[\"error\"] += 1\n                errors.append(f\"{futs[f]}\\t{st}\")\n            else:\n                counts[st] += 1\n\n    dst.mkdir(parents=True, exist_ok=True)\n    out_bytes = sum(p.stat().st_size for p in dst.rglob(\"*\") if p.is_file())\n    (dst / \"_manifest.json\").write_text(json.dumps({\n        \"source\": str(src), \"target\": target,\n        \"mode\": \"square\" if square else \"shortest_edge\",\n        \"jpeg_quality\": quality, \"subsampling\": \"4:4:4\",\n        \"resampling\": \"BICUBIC\" if square else \"LANCZOS\",\n        \"counts\": counts, \"total\": len(jobs), \"output_bytes\": out_bytes,\n        \"built_at\": time.strftime(\"%Y-%m-%dT%H:%M:%S\"),\n    }, indent=2), encoding=\"utf-8\")\n    if errors:\n        (dst / \"_errors.txt\").write_text(\"\\n\".join(errors), encoding=\"utf-8\")\n        print(f\"[resize] WARNING: {len(errors)} failures -> {dst/'_errors.txt'}\")\n    print(f\"[resize] done: {counts}\")\n    print(f\"[resize] output size: {out_bytes/1024**3:.2f} GB \"\n          f\"({out_bytes/max(1,len(jobs))/1024:.0f} KB/image avg)\")\n\n\ndef pack_shards(dst: Path, shards_dir: Path, shard_gb, prefix=\"cxr\"):\n    shard_bytes = int(shard_gb * 1024**3)\n    shards_dir.mkdir(parents=True, exist_ok=True)\n    files = sorted(p for p in dst.rglob(\"*\")\n                   if p.is_file() and p.name not in (\"_manifest.json\", \"_errors.txt\"))\n    if not files:\n        raise SystemExit(f\"ERROR: nothing to pack under {dst}\")\n    print(f\"[pack] {len(files):,} files -> tar shards (~{shard_gb} GB each)\")\n    written, idx, cur = [], 0, 0\n\n    def _open(i):\n        path = shards_dir / f\"{prefix}-{i:04d}.tar\"\n        written.append(path)\n        return tarfile.open(path, \"w\")\n\n    tar = _open(0)\n    for fp in tqdm(files, unit=\"file\"):\n        if cur >= shard_bytes:\n            tar.close(); idx += 1; tar = _open(idx); cur = 0\n        tar.add(fp, arcname=str(fp.relative_to(dst)))   # rel path -> tree rebuilt on extract\n        cur += fp.stat().st_size\n    tar.close()\n    man = dst / \"_manifest.json\"\n    if man.exists():\n        shutil.copy2(man, shards_dir / \"_manifest.json\")\n    (shards_dir / \"SHARDS.txt\").write_text(\"\\n\".join(p.name for p in written), encoding=\"utf-8\")\n    print(f\"[pack] wrote {len(written)} shards -> {shards_dir}\")\n    return written\n\nprint(\"functions ready\")"
+   "source": "import os, json, shutil, tarfile, time\nfrom pathlib import Path\nfrom concurrent.futures import ThreadPoolExecutor, as_completed\nfrom PIL import Image\nfrom tqdm.auto import tqdm\n\nImage.MAX_IMAGE_PIXELS = None          # don't abort on large medical images\nIMG_EXTS = (\".jpg\", \".jpeg\", \".png\")\n\n\ndef _resize_one(src_path, dst_path, target, quality, square):\n    \"\"\"Returns one of: resized | squared | copied | skipped | error:<msg>.\"\"\"\n    try:\n        dst_path = Path(dst_path)\n        if dst_path.exists() and dst_path.stat().st_size > 0:\n            return \"skipped\"                       # resumable\n        dst_path.parent.mkdir(parents=True, exist_ok=True)\n        with Image.open(src_path) as im:\n            w, h = im.size\n            shorter = min(w, h)\n            # Non-square: if shorter side already <= target, downscaling would\n            # push it below 518 -> copy verbatim (lossless, never worsens a\n            # low-res source). Square mode must always emit exactly target^2.\n            if not square and shorter <= target:\n                shutil.copy2(src_path, dst_path)\n                return \"copied\"\n            if im.mode not in (\"L\", \"RGB\"):\n                im = im.convert(\"RGB\")\n            # shorter axis EXACTLY = target; longer scales proportionally\n            if w <= h:\n                new_size = (target, round(h * target / w))\n            else:\n                new_size = (round(w * target / h), target)\n            # square mode reproduces the processor exactly -> bicubic\n            im = im.resize(new_size, Image.BICUBIC if square else Image.LANCZOS)\n            if square:\n                W, H = im.size\n                left, top = (W - target) // 2, (H - target) // 2\n                im = im.crop((left, top, left + target, top + target))\n            im.save(dst_path, \"JPEG\", quality=quality, optimize=True, subsampling=0)\n        return \"squared\" if square else \"resized\"\n    except Exception as e:\n        return f\"error:{type(e).__name__}: {e}\"\n\n\ndef _copy_one(src_path, dst_path):\n    \"\"\"Copy non-image files (reports .txt, chexpert .csv, metadata .json, ...)\n    verbatim so the shipped tree mirrors MIMIC-CXR_processed exactly.\"\"\"\n    try:\n        dst_path = Path(dst_path)\n        if dst_path.exists() and dst_path.stat().st_size > 0:\n            return \"skipped\"\n        dst_path.parent.mkdir(parents=True, exist_ok=True)\n        shutil.copy2(src_path, dst_path)\n        return \"copied_other\"\n    except Exception as e:\n        return f\"error:{type(e).__name__}: {e}\"\n\n\ndef resize_tree(src: Path, dst: Path, target, quality, workers, square):\n    print(f\"[resize] scanning {src} ...\")\n    img_jobs, other_jobs = [], []\n    for root, _, files in os.walk(src):\n        for fn in files:\n            sp = Path(root) / fn\n            rel = sp.relative_to(src)\n            dp = dst / rel\n            if fn.lower().endswith(IMG_EXTS):\n                img_jobs.append((str(sp), str(dp)))\n            else:\n                other_jobs.append((str(sp), str(dp)))\n    if not img_jobs and not other_jobs:\n        raise SystemExit(f\"ERROR: nothing found under {src}\")\n    mode = f\"square {target}x{target}\" if square else f\"shortest-edge {target}px\"\n    print(f\"[resize] {len(img_jobs):,} images + {len(other_jobs):,} non-image \"\n          f\"-> {dst}  ({mode}, q{quality}, {workers} threads)\")\n\n    counts = {\"resized\": 0, \"squared\": 0, \"copied\": 0,\n              \"copied_other\": 0, \"skipped\": 0, \"error\": 0}\n    errors = []\n    with ThreadPoolExecutor(max_workers=workers) as ex:\n        futs = {}\n        for s, d in img_jobs:\n            futs[ex.submit(_resize_one, s, d, target, quality, square)] = d\n        for s, d in other_jobs:\n            futs[ex.submit(_copy_one, s, d)] = d\n        for f in tqdm(as_completed(futs), total=len(futs), unit=\"file\"):\n            st = f.result()\n            if st.startswith(\"error:\"):\n                counts[\"error\"] += 1\n                errors.append(f\"{futs[f]}\\t{st}\")\n            else:\n                counts[st] += 1\n\n    dst.mkdir(parents=True, exist_ok=True)\n    total = len(img_jobs) + len(other_jobs)\n    out_bytes = sum(p.stat().st_size for p in dst.rglob(\"*\") if p.is_file())\n    (dst / \"_manifest.json\").write_text(json.dumps({\n        \"source\": str(src), \"target\": target,\n        \"mode\": \"square\" if square else \"shortest_edge\",\n        \"jpeg_quality\": quality, \"subsampling\": \"4:4:4\",\n        \"resampling\": \"BICUBIC\" if square else \"LANCZOS\",\n        \"counts\": counts, \"total\": total,\n        \"images\": len(img_jobs), \"non_image\": len(other_jobs),\n        \"output_bytes\": out_bytes,\n        \"built_at\": time.strftime(\"%Y-%m-%dT%H:%M:%S\"),\n    }, indent=2), encoding=\"utf-8\")\n    if errors:\n        (dst / \"_errors.txt\").write_text(\"\\n\".join(errors), encoding=\"utf-8\")\n        print(f\"[resize] WARNING: {len(errors)} failures -> {dst/'_errors.txt'}\")\n    print(f\"[resize] done: {counts}\")\n    print(f\"[resize] output size: {out_bytes/1024**3:.2f} GB \"\n          f\"({out_bytes/max(1,len(img_jobs))/1024:.0f} KB/image avg)\")\n\n\ndef pack_shards(dst: Path, shards_dir: Path, shard_gb, prefix=\"cxr\"):\n    shard_bytes = int(shard_gb * 1024**3)\n    shards_dir.mkdir(parents=True, exist_ok=True)\n    files = sorted(p for p in dst.rglob(\"*\")\n                   if p.is_file() and p.name not in (\"_manifest.json\", \"_errors.txt\"))\n    if not files:\n        raise SystemExit(f\"ERROR: nothing to pack under {dst}\")\n    print(f\"[pack] {len(files):,} files -> tar shards (~{shard_gb} GB each)\")\n    written, idx, cur = [], 0, 0\n\n    def _open(i):\n        path = shards_dir / f\"{prefix}-{i:04d}.tar\"\n        written.append(path)\n        return tarfile.open(path, \"w\")\n\n    tar = _open(0)\n    for fp in tqdm(files, unit=\"file\"):\n        if cur >= shard_bytes:\n            tar.close(); idx += 1; tar = _open(idx); cur = 0\n        tar.add(fp, arcname=str(fp.relative_to(dst)))   # rel path -> tree rebuilt on extract\n        cur += fp.stat().st_size\n    tar.close()\n    man = dst / \"_manifest.json\"\n    if man.exists():\n        shutil.copy2(man, shards_dir / \"_manifest.json\")\n    (shards_dir / \"SHARDS.txt\").write_text(\"\\n\".join(p.name for p in written), encoding=\"utf-8\")\n    print(f\"[pack] wrote {len(written)} shards -> {shards_dir}\")\n    return written\n\nprint(\"functions ready\")"
   },
   {
    "cell_type": "markdown",
@@ -137,4 +137,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}