File size: 8,037 Bytes
80e070e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | """
RockGlacierNet β single source of truth for DATA COLLECTION + PATCH BUILDING.
Everything the collection/patching pipeline writes lives under `patch_dataset/`.
Edit values here; no other file needs touching to re-collect at a different
patch size or with a different context window.
Pipeline (Option A β enlarged per-glacier window):
1. preprocess/collect_features.py shapefile -> patch_dataset/raw/{features,masks} (run on Windows: needs ee+geopandas)
2. preprocess/build_patches.py raw/ -> patch_dataset/patches_{size}/{split} (run anywhere: needs rasterio)
Why Option A: the old 200 m buffer produced tiles whose median size is 83 px, so
86% were smaller than a 128 px patch β a 128 patch on them would be mostly
reflect-padding. We instead download a generous context window ONCE (big enough
for the largest target patch) and tile locally to any size in PATCH_SIZES.
"""
from pathlib import Path
# ββ Roots βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# config.py lives in <project>/patch_dataset/ , so:
PATCH_DATASET_DIR = Path(__file__).resolve().parent # .../patch_dataset
PROJECT_ROOT = PATCH_DATASET_DIR.parent # .../RockGlacierNet
# Input shapefile (rock-glacier inventory polygons). Override on Windows.
SHAPEFILE_PATH = PROJECT_ROOT / "Final_merged01" / "Final_merged01.shp"
# All COLLECTED + BUILT artifacts stay inside patch_dataset/ (and nowhere else).
RAW_DIR = PATCH_DATASET_DIR / "raw"
RAW_FEATURES_DIR = RAW_DIR / "features" # feature_XXXX.tif (12-band) [per-glacier, legacy]
RAW_MASKS_DIR = RAW_DIR / "masks" # mask_XXXX.tif (1-band, pixel-aligned)
# Region pipeline (whole-Sikkim grid) output β used by collect_region.py.
RAW_REGION_DIR = PATCH_DATASET_DIR / "raw_region"
RAW_REGION_FEATURES_DIR = RAW_REGION_DIR / "features" # tile_RRRR_CCCC.tif (12-band)
RAW_REGION_MASKS_DIR = RAW_REGION_DIR / "masks" # tile_RRRR_CCCC.tif (1-band, aligned)
# Enlarged per-glacier pipeline output β used by collect_glacier.py.
# One crop per inventory polygon, glacier centered, with real context (not padding).
RAW_GLACIER_DIR = PATCH_DATASET_DIR / "raw_glacier"
RAW_GLACIER_FEATURES_DIR = RAW_GLACIER_DIR / "features" # feature_XXXX.tif (12-band)
RAW_GLACIER_MASKS_DIR = RAW_GLACIER_DIR / "masks" # mask_XXXX.tif (1-band, aligned)
# Buffer added around each polygon bbox. 400 m makes a typical crop ~128 px: the
# glacier stays prominent (~10-15% positive, like the working dataset/) while its
# border is REAL terrain instead of the zero-padding the old dataset/ used.
GLACIER_BUFFER_M = 400.0
# ββ Geometry / projection βββββββββββββββββββββββββββββββββββββββββββββββββββββ
TARGET_CRS = "EPSG:32645" # UTM Zone 45N (metres) β Sikkim
SCALE_M = 10.0 # output pixel size (metres). Sentinel-2 native 10 m.
# ββ Patch sizes βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
PATCH_SIZES = [64, 128, 256] # sizes you may build from the SAME raw download
DEFAULT_PATCH = 128 # used when build_patches.py is run with no --patch-size
# Context window downloaded per glacier (metres added on every side of the
# polygon bbox). Must be >= (max(PATCH_SIZES)/2)*SCALE_M so even a tiny glacier
# yields a tile large enough to hold the biggest patch. For 256 px that is 1280 m.
CONTEXT_BUFFER_M = (max(PATCH_SIZES) // 2) * SCALE_M + 50.0 # = 1330 m
# ββ Region (whole-Sikkim) collection ββββββββββββββββββββββββββββββββββββββββββ
# AOI = bounding box of all inventory polygons (= essentially the Sikkim glacier
# belt, ~7130 km^2), expanded by AOI_MARGIN_M. The AOI is gridded into square
# download tiles of RAW_TILE_PX px; each tile is downloaded as one 12-band GeoTIFF.
AOI_MARGIN_M = 1000.0 # expand the glacier-bbox AOI by this on every side
RAW_TILE_PX = 512 # download-tile size (px). 512 @ 10 m = 5.12 km, ~8 MB/tile (safe < 32 MB GEE limit)
# Which grid tiles to download:
# "sampled" β every tile touching a glacier + REGION_BG_KEEP of background tiles (good training set)
# "dense" β every tile in the AOI (full Sikkim mosaic; for region-wide inference). Multi-GB.
REGION_MODE = "sampled"
REGION_BG_KEEP = 0.25 # (sampled mode) fraction of pure-background tiles to also download
S2_CLOUD_PCT = 20 # scene-level CLOUDY_PIXEL_PERCENTAGE filter β keeps the median composite
# light enough to avoid GEE "User memory limit exceeded" on download.
# ββ Patch extraction ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
STRIDE_FRACTION = 0.5 # patch overlap; stride = round(patch_size * this)
BACKGROUND_KEEP_RATIO = 0.05 # fraction of pure-background patches to keep
MIN_GLACIER_PERCENTAGE = 2.0 # keep any patch with > this % glacier pixels
PADDING_MODE = "reflect" # "reflect" | "constant" (0-fill)
# ββ Train / val / test split (by SOURCE GLACIER β prevents patch leakage) βββββ
SPLIT_RATIOS = {"train": 0.70, "val": 0.15, "test": 0.15}
RANDOM_SEED = 42
# ββ Bands βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
# ORDER IS CONTRACTUAL: must match data_generator.GLOBAL_MEAN / GLOBAL_STD.
NUM_BANDS = 12
BAND_NAMES = ["B2", "B3", "B4", "B8", "B11", "B12",
"NDVI", "NDWI", "NDSI", "Elev", "Slope", "Aspect"]
# ββ Earth Engine collection settings (used by collect_features.py) ββββββββββββ
GEE_PROJECT = "rock-glacier-net"
S2_COLLECTION = "COPERNICUS/S2_SR_HARMONIZED"
DEM_COLLECTION = "COPERNICUS/DEM/GLO30"
S2_DATE_START = "2021-01-01"
S2_DATE_END = "2024-01-01"
S2_SR_DIVISOR = 10000.0 # scale Sentinel-2 SR DN -> reflectance
NODATA_VALUE = -9999
# ββ Derived helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
def stride_for(patch_size: int) -> int:
"""Step between patch origins for a given patch size."""
return max(1, round(patch_size * STRIDE_FRACTION))
def patches_dir(patch_size: int) -> Path:
"""Output root for a given patch size, e.g. patch_dataset/patches_128/."""
return PATCH_DATASET_DIR / f"patches_{patch_size}"
def as_dict(patch_size: int) -> dict:
"""Flat config snapshot written into each build's metadata/config.json."""
return {
"patch_size": patch_size,
"stride": stride_for(patch_size),
"stride_fraction": STRIDE_FRACTION,
"scale_m": SCALE_M,
"context_buffer_m": CONTEXT_BUFFER_M,
"background_keep_ratio": BACKGROUND_KEEP_RATIO,
"minimum_glacier_pct": MIN_GLACIER_PERCENTAGE,
"padding_mode": PADDING_MODE,
"split_ratios": SPLIT_RATIOS,
"random_seed": RANDOM_SEED,
"target_crs": TARGET_CRS,
"num_bands": NUM_BANDS,
"band_names": BAND_NAMES,
"raw_features_dir": str(RAW_FEATURES_DIR),
"raw_masks_dir": str(RAW_MASKS_DIR),
"output_dir": str(patches_dir(patch_size)),
}
|