{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Rock Glacier Dataset: Pre-Training Data Validation & EDA\n", "\n", "This notebook validates the alignment between our 12-band Earth Engine features and the binary masks, and checks data quality before deep learning training.\n", "\n", "**Dataset Specs:**\n", "- CRS: `EPSG:32645` (UTM Zone 45N)\n", "- Resolution: 10m/pixel\n", "- Features: 12-band GeoTIFF (Float32/Float64)\n", "- Masks: 1-band GeoTIFF (Binary: 0=Background, 1=Rock Glacier)\n", "\n", "**12-Band Feature Mapping:**\n", "\n", "| Band (1-indexed) | Name | Description |\n", "|:-----------------:|:------------|:--------------------------------|\n", "| 1 | B2 | Blue |\n", "| 2 | B3 | Green |\n", "| 3 | B4 | Red |\n", "| 4 | B8 | Near-Infrared (NIR) |\n", "| 5 | B11 | SWIR-1 |\n", "| 6 | B12 | SWIR-2 |\n", "| 7 | NDVI | Vegetation Index |\n", "| 8 | NDWI | Water Index |\n", "| 9 | NDSI | Snow Index |\n", "| 10 | Elevation | Copernicus DEM GLO-30 (meters) |\n", "| 11 | Slope | Terrain slope (degrees) |\n", "| 12 | Aspect | Terrain aspect (degrees) |" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1. Environment Setup & Initialization" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Found 607 feature files.\n", "Found 610 mask files.\n" ] } ], "source": [ "import os\n", "import glob\n", "import rasterio\n", "import geopandas as gpd\n", "import numpy as np\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import matplotlib.patches as mpatches\n", "import seaborn as sns\n", "from rasterio.plot import show\n", "from scipy import stats\n", "import warnings\n", "warnings.filterwarnings('ignore')\n", "\n", "# ── Directory Configuration ──────────────────────────────────────────────────\n", "FEATURE_DIR = r'D:\\RockGlacier_Project\\features'\n", "MASK_DIR = r'D:\\RockGlacier_Project\\masks'\n", "POLYGON_DIR = r'D:\\RockGlacier_Project\\EDA\\individual_polygons'\n", "\n", "# ── Band name lookup (1-indexed, matching rasterio .read(band)) ───────────────\n", "BAND_NAMES = {\n", " 1: 'B2 (Blue)',\n", " 2: 'B3 (Green)',\n", " 3: 'B4 (Red)',\n", " 4: 'B8 (NIR)',\n", " 5: 'B11 (SWIR-1)',\n", " 6: 'B12 (SWIR-2)',\n", " 7: 'NDVI',\n", " 8: 'NDWI',\n", " 9: 'NDSI',\n", " 10: 'Elevation (m)',\n", " 11: 'Slope (°)',\n", " 12: 'Aspect (°)',\n", "}\n", "\n", "# ── Discover files ────────────────────────────────────────────────────────────\n", "feature_files = sorted(glob.glob(os.path.join(FEATURE_DIR, 'feature_*.tif')))\n", "mask_files = sorted(glob.glob(os.path.join(MASK_DIR, 'mask_*.tif')))\n", "\n", "print(f\"Found {len(feature_files)} feature files.\")\n", "print(f\"Found {len(mask_files)} mask files.\")\n", "\n", "# ── Helper: extract zero-padded index from filename ───────────────────────────\n", "def get_idx(filepath):\n", " \"\"\"Return the XXXX index string from 'feature_XXXX.tif' or 'mask_XXXX.tif'.\"\"\"\n", " return os.path.basename(filepath).split('_')[1].split('.')[0]\n", "\n", "# Build a lookup: idx -> mask_path\n", "mask_lookup = {get_idx(p): p for p in mask_files}" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Spatial Alignment Validation\n", "\n", "Checks every `feature_XXXX.tif` / `mask_XXXX.tif` pair for identical `.shape`, `.crs`, and `.transform`.\n", "A mismatch here will cause silent tensor corruption in a PyTorch `Dataset` — catch it now." ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Validating spatial alignment across all downloaded pairs...\n", "\n", "✅ Aligned pairs : 0\n", "❌ Mismatched pairs : 607\n", "⚠️ Missing masks : 0\n", "\n", "Mismatch details:\n" ] }, { "data": { "text/html": [ "
| \n", " | file | \n", "feature_shape | \n", "mask_shape | \n", "feature_crs | \n", "mask_crs | \n", "shape_match | \n", "crs_match | \n", "transform_match | \n", "feature_bands | \n", "mask_bands | \n", "
|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", "feature_0000.tif | \n", "(95, 84) | \n", "(92, 81) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 1 | \n", "feature_0001.tif | \n", "(87, 137) | \n", "(83, 134) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 2 | \n", "feature_0002.tif | \n", "(56, 52) | \n", "(53, 49) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 3 | \n", "feature_0003.tif | \n", "(74, 76) | \n", "(72, 73) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 4 | \n", "feature_0004.tif | \n", "(140, 137) | \n", "(136, 133) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 602 | \n", "feature_0620.tif | \n", "(62, 61) | \n", "(59, 58) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 603 | \n", "feature_0621.tif | \n", "(62, 67) | \n", "(59, 64) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 604 | \n", "feature_0622.tif | \n", "(53, 65) | \n", "(50, 62) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 605 | \n", "feature_0623.tif | \n", "(71, 64) | \n", "(68, 61) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
| 606 | \n", "feature_0624.tif | \n", "(59, 59) | \n", "(56, 55) | \n", "EPSG:32645 | \n", "EPSG:32645 | \n", "False | \n", "True | \n", "False | \n", "12 | \n", "1 | \n", "
607 rows × 10 columns
\n", "| \n", " | name | \n", "min | \n", "max | \n", "mean | \n", "std | \n", "p5 | \n", "p95 | \n", "
|---|---|---|---|---|---|---|---|
| band | \n", "\n", " | \n", " | \n", " | \n", " | \n", " | \n", " | \n", " |
| 1 | \n", "B2 (Blue) | \n", "0.0342 | \n", "1.0108 | \n", "0.2902 | \n", "0.1748 | \n", "0.1088 | \n", "0.6385 | \n", "
| 2 | \n", "B3 (Green) | \n", "0.0497 | \n", "0.9814 | \n", "0.3038 | \n", "0.1619 | \n", "0.1299 | \n", "0.6245 | \n", "
| 3 | \n", "B4 (Red) | \n", "0.0649 | \n", "0.9576 | \n", "0.3145 | \n", "0.1528 | \n", "0.1522 | \n", "0.6198 | \n", "
| 4 | \n", "B8 (NIR) | \n", "0.0976 | \n", "0.8865 | \n", "0.3347 | \n", "0.1261 | \n", "0.2055 | \n", "0.5964 | \n", "
| 5 | \n", "B11 (SWIR-1) | \n", "0.0603 | \n", "0.4996 | \n", "0.2761 | \n", "0.0492 | \n", "0.1935 | \n", "0.3477 | \n", "
| 6 | \n", "B12 (SWIR-2) | \n", "0.0644 | \n", "0.4637 | \n", "0.2518 | \n", "0.0434 | \n", "0.1818 | \n", "0.3202 | \n", "
| 7 | \n", "NDVI | \n", "-0.2525 | \n", "0.5095 | \n", "0.0523 | \n", "0.0760 | \n", "-0.0286 | \n", "0.1987 | \n", "
| 8 | \n", "NDWI | \n", "-0.5650 | \n", "0.3116 | \n", "-0.0799 | \n", "0.1000 | \n", "-0.2638 | \n", "0.0345 | \n", "
| 9 | \n", "NDSI | \n", "-0.6708 | \n", "0.8516 | \n", "-0.0115 | \n", "0.2808 | \n", "-0.3916 | \n", "0.4613 | \n", "
| 10 | \n", "Elevation (m) | \n", "4478.2305 | \n", "5918.7007 | \n", "5144.2652 | \n", "331.2679 | \n", "4658.2959 | \n", "5647.1550 | \n", "
| 11 | \n", "Slope (°) | \n", "0.0000 | \n", "68.9936 | \n", "17.9009 | \n", "10.4299 | \n", "3.2059 | \n", "35.9434 | \n", "
| 12 | \n", "Aspect (°) | \n", "-0.0000 | \n", "359.9875 | \n", "161.0213 | \n", "126.3876 | \n", "8.8786 | \n", "351.8176 | \n", "