| """ |
| RockGlacierNet — Build patches from region tiles. [lives in patch_dataset/] |
| |
| raw_region/{features,masks} -> patch_dataset/patches_{size}/{split}/{features,masks}/ |
| |
| * pairs each feature tile with its mask (region tiles share the same name; |
| legacy per-glacier feature_/mask_ pairs also work), |
| * assigns each SOURCE TILE to train/val/test (split by tile -> no |
| overlapping-patch leakage within a tile), |
| * tiles every source into patch_size x patch_size GeoTIFF patches with overlap, |
| keeping all glacier patches + a sampled fraction of background patches, |
| * writes georeferenced GeoTIFF patches named feature_*.tif / mask_*.tif so the |
| existing RockGlacierDataGenerator trains on them unchanged, |
| * writes metadata/ (config.json, patch_metadata.csv, dataset_summary.csv, |
| diagnostics_report.txt). |
| |
| Run (rasterio only): |
| python patch_dataset/build_patches.py # DEFAULT_PATCH, from raw_region/ |
| python patch_dataset/build_patches.py --patch-size 64 |
| python patch_dataset/build_patches.py --patch-size 256 |
| All tunables live in patch_dataset/config.py. |
| """ |
|
|
| import os |
| import sys |
| import glob |
| import json |
| import math |
| import argparse |
| import numpy as np |
| import rasterio |
| import pandas as pd |
| from rasterio.windows import Window, transform as window_transform |
|
|
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) |
| import config as C |
|
|
|
|
| |
| |
| |
|
|
| def padded_size(dim, patch, stride): |
| if dim <= patch: |
| return patch |
| return math.ceil((dim - patch) / stride) * stride + patch |
|
|
|
|
| def iter_patches(feat, mask, patch, stride, mode): |
| H, W = feat.shape[:2] |
| pH, pW = padded_size(H, patch, stride), padded_size(W, patch, stride) |
| dh, dw = pH - H, pW - W |
| padded = dh > 0 or dw > 0 |
| if padded: |
| feat = np.pad(feat, ((0, dh), (0, dw), (0, 0)), |
| mode=mode if mode == "reflect" else "constant") |
| mask = np.pad(mask, ((0, dh), (0, dw), (0, 0)), |
| mode="constant", constant_values=0) |
| for r in range(0, pH - patch + 1, stride): |
| for c in range(0, pW - patch + 1, stride): |
| yield feat[r:r+patch, c:c+patch, :], mask[r:r+patch, c:c+patch, :], r, c, padded |
|
|
|
|
| def assign_splits(ids, ratios, seed): |
| rng = np.random.RandomState(seed) |
| ids = sorted(ids); rng.shuffle(ids) |
| n = len(ids); n_tr = int(n * ratios["train"]); n_va = int(n * (ratios["train"] + ratios["val"])) |
| return {iid: ("train" if i < n_tr else ("val" if i < n_va else "test")) for i, iid in enumerate(ids)} |
|
|
|
|
| def _mask_path_for(feat_path, mask_dir): |
| """Region tiles share the name; legacy uses feature_/mask_ prefixes.""" |
| base = os.path.basename(feat_path) |
| for cand in (base, base.replace("feature_", "mask_"), base.replace("tile_", "tile_")): |
| p = os.path.join(mask_dir, cand) |
| if os.path.exists(p): |
| return p |
| return os.path.join(mask_dir, base) |
|
|
|
|
| def _id_of(feat_path): |
| stem = os.path.splitext(os.path.basename(feat_path))[0] |
| return stem.replace("feature_", "").replace("tile_", "") |
|
|
|
|
| |
| |
| |
|
|
| def run(patch, feat_dir, mask_dir, limit): |
| stride = C.stride_for(patch) |
| outdir = C.patches_dir(patch) |
| cfg = C.as_dict(patch) |
| cfg["features_dir_used"], cfg["masks_dir_used"] = str(feat_dir), str(mask_dir) |
|
|
| (outdir / "metadata").mkdir(parents=True, exist_ok=True) |
| for sp in C.SPLIT_RATIOS: |
| (outdir / sp / "features").mkdir(parents=True, exist_ok=True) |
| (outdir / sp / "masks").mkdir(parents=True, exist_ok=True) |
| with open(outdir / "metadata" / "config.json", "w") as f: |
| json.dump(cfg, f, indent=2) |
|
|
| feat_files = sorted(glob.glob(os.path.join(str(feat_dir), "*.tif"))) |
| if limit: |
| feat_files = feat_files[:limit] |
| ids = [_id_of(fp) for fp in feat_files] |
| split_of = assign_splits(ids, C.SPLIT_RATIOS, C.RANDOM_SEED) |
| print(f"patch_size={patch} stride={stride} | {len(feat_files)} source tiles -> {outdir.name}/") |
|
|
| rng = np.random.RandomState(C.RANDOM_SEED) |
| patch_records, image_records = [], [] |
| n_gen = n_kept = 0 |
|
|
| for fp in feat_files: |
| iid = _id_of(fp); split = split_of[iid] |
| mp = _mask_path_for(fp, str(mask_dir)) |
| if not os.path.exists(mp): |
| print(f" {iid}: no matching mask — skipped"); continue |
|
|
| with rasterio.open(fp) as fs: |
| feat = np.moveaxis(fs.read().astype(np.float32), 0, -1) |
| base_transform, crs = fs.transform, fs.crs |
| with rasterio.open(mp) as ms: |
| mask = ms.read(1).astype(np.uint8)[:, :, np.newaxis] |
|
|
| ch, cw = min(feat.shape[0], mask.shape[0]), min(feat.shape[1], mask.shape[1]) |
| feat, mask = feat[:ch, :cw, :], mask[:ch, :cw, :] |
| feat = np.where((feat == C.NODATA_VALUE) | ~np.isfinite(feat), 0.0, feat) |
|
|
| oH, oW = feat.shape[:2]; img_fg = int((mask > 0).sum()) |
| n_img_kept = n_img_gen = 0 |
|
|
| for fpat, mpat, r, c, padded in iter_patches(feat, mask, patch, stride, C.PADDING_MODE): |
| g_pct = 100.0 * int((mpat > 0).sum()) / mpat.size |
| if g_pct > C.MIN_GLACIER_PERCENTAGE: |
| keep = True |
| elif g_pct == 0.0: |
| keep = (rng.rand() < C.BACKGROUND_KEEP_RATIO) |
| else: |
| keep = False |
| n_gen += 1; n_img_gen += 1 |
| patch_id = f"{iid}_r{r:04d}_c{c:04d}" |
| rec = {"patch_id": patch_id, "source_image_id": iid, "split": split, |
| "patch_row": r, "patch_col": c, |
| "glacier_pixel_count": int((mpat > 0).sum()), |
| "glacier_percentage": round(g_pct, 4), |
| "kept_or_filtered": "kept" if keep else "filtered", |
| "padding_applied": bool(padded)} |
| for b, name in enumerate(C.BAND_NAMES): |
| rec[f"mean_{name}"] = round(float(fpat[:, :, b].mean()), 6) |
| patch_records.append(rec) |
| if not keep: |
| continue |
| ptransform = window_transform(Window(c, r, patch, patch), base_transform) |
| fout = outdir / split / "features" / f"feature_{patch_id}.tif" |
| mout = outdir / split / "masks" / f"mask_{patch_id}.tif" |
| with rasterio.open(fout, "w", driver="GTiff", height=patch, width=patch, |
| count=C.NUM_BANDS, dtype="float32", crs=crs, transform=ptransform) as dst: |
| dst.write(np.moveaxis(fpat.astype(np.float32), -1, 0)) |
| with rasterio.open(mout, "w", driver="GTiff", height=patch, width=patch, |
| count=1, dtype="uint8", crs=crs, transform=ptransform, nodata=0) as dst: |
| dst.write(mpat[:, :, 0].astype(np.uint8), 1) |
| n_kept += 1; n_img_kept += 1 |
|
|
| image_records.append({"image_id": iid, "split": split, |
| "original_height": oH, "original_width": oW, |
| "glacier_pixels": img_fg, |
| "glacier_percentage": round(100.0 * img_fg / mask.size, 4), |
| "generated_patches": n_img_gen, "kept_patches": n_img_kept}) |
|
|
| patch_df, image_df = pd.DataFrame(patch_records), pd.DataFrame(image_records) |
| patch_df.to_csv(outdir / "metadata" / "patch_metadata.csv", index=False) |
| image_df.to_csv(outdir / "metadata" / "dataset_summary.csv", index=False) |
| _diagnostics(patch_df, image_df, outdir, cfg) |
| print(f"\nDone. generated={n_gen} kept={n_kept} -> {outdir}") |
|
|
|
|
| def _diagnostics(patch_df, image_df, outdir, cfg): |
| kept = patch_df[patch_df["kept_or_filtered"] == "kept"] if len(patch_df) else patch_df |
| L = [] |
| def h(t): L.extend(["", "─" * 56, f" {t}", "─" * 56]) |
| L += ["=" * 56, " ROCKGLACIERENET — PATCH BUILD DIAGNOSTICS", "=" * 56, |
| f" patch_size {cfg['patch_size']} | stride {cfg['stride']} | " |
| f"bg_keep {cfg['background_keep_ratio']} | pad {cfg['padding_mode']}"] |
| h("PATCH COUNTS") |
| for sp in C.SPLIT_RATIOS: |
| g = len(patch_df[patch_df["split"] == sp]) if len(patch_df) else 0 |
| k = len(kept[kept["split"] == sp]) if len(kept) else 0 |
| L.append(f" {sp:5s}: generated={g:6d} kept={k:6d} filtered={g-k:6d}") |
| L.append(f" TOTAL: generated={len(patch_df):6d} kept={len(kept):6d}") |
| h("SOURCE TILES PER SPLIT") |
| for sp in C.SPLIT_RATIOS: |
| L.append(f" {sp:5s}: {len(image_df[image_df['split']==sp]) if len(image_df) else 0} tiles") |
| if len(kept): |
| h("PIXEL CLASS BALANCE (kept patches)") |
| tfg = int(kept["glacier_pixel_count"].sum()); tpx = len(kept) * cfg["patch_size"] ** 2 |
| tbg = tpx - tfg |
| L.append(f" Glacier : {tfg:>12,} ({100*tfg/tpx:.2f}%)") |
| L.append(f" Background: {tbg:>12,} ({100*tbg/tpx:.2f}%)") |
| if tfg: L.append(f" Imbalance : {tbg/tfg:.1f} : 1") |
| h("GLACIER COVERAGE OF KEPT PATCHES (%)") |
| for lo, hi in [(0, 0.1), (0.1, 5), (5, 20), (20, 50), (50, 80), (80, 100.01)]: |
| n = int(((kept["glacier_percentage"] >= lo) & (kept["glacier_percentage"] < hi)).sum()) |
| L.append(f" {lo:5.1f}-{hi:5.1f}%: {n:6d} {'█' * min(n // 5, 40)}") |
| report = "\n".join(L) |
| print("\n" + report) |
| with open(outdir / "metadata" / "diagnostics_report.txt", "w") as f: |
| f.write(report) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser(description="Build GeoTIFF patches from region tiles.") |
| ap.add_argument("--patch-size", type=int, default=C.DEFAULT_PATCH) |
| ap.add_argument("--features-dir", default=str(C.RAW_REGION_FEATURES_DIR)) |
| ap.add_argument("--masks-dir", default=str(C.RAW_REGION_MASKS_DIR)) |
| ap.add_argument("--limit", type=int, default=None) |
| args = ap.parse_args() |
| run(args.patch_size, args.features_dir, args.masks_dir, args.limit) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
