| """ |
| Ensemble evaluation for RockGlacierNet — averages N SwinUNet checkpoints. |
| |
| Each model runs 8-way TTA (4 rotations x 2 flips); the per-pixel sigmoid |
| probabilities are averaged across ALL models AND all TTA transforms, then |
| scored exactly like evaluate.py (valid-region only, threshold sweep, per-image). |
| |
| Usage (set the checkpoints, then run): |
| CKPT_PATHS="dir1/best.h5,dir2/best.h5,dir3/best.h5" \ |
| DATA_DIR=dataset/patches_v2 \ |
| BAND_LIST="band_25,band_18,..." \ |
| OUT_DIR=bandselection/ensemble_test \ |
| python3 evaluate_ensemble.py |
| """ |
|
|
| import os |
| import glob |
| import json |
| import numpy as np |
| np.Inf = np.inf |
| import pandas as pd |
| import tensorflow as tf |
| from sklearn.metrics import confusion_matrix |
|
|
| from data_generator import GLOBAL_MEAN, GLOBAL_STD |
| from models.swin_unet import build_model |
|
|
| |
| |
| |
| DATA_DIR = os.environ.get("DATA_DIR", "dataset/patches_v2") |
| TEST_FEAT = os.environ.get("TEST_FEAT", f"{DATA_DIR}/test/features") |
| TEST_MASK = os.environ.get("TEST_MASK", f"{DATA_DIR}/test/masks") |
| OUT_DIR = os.environ.get("OUT_DIR", "bandselection/ensemble_test") |
|
|
| |
| CKPT_PATHS = [p.strip() for p in os.environ["CKPT_PATHS"].split(",") if p.strip()] |
|
|
| |
| _band_names = json.load(open(f"{DATA_DIR}/metadata/norm_stats.json"))["band_names"] |
| if os.environ.get("BAND_LIST"): |
| _bands = [b.strip() for b in os.environ["BAND_LIST"].split(",") if b.strip()] |
| BAND_INDICES = [_band_names.index(b) for b in _bands] |
| else: |
| BAND_INDICES = None |
|
|
| _ph, _pw = np.load(sorted(glob.glob(f"{TEST_FEAT}/*.npy"))[0]).shape[:2] |
| N_BANDS = len(BAND_INDICES) if BAND_INDICES else len(_band_names) |
| INPUT_SHAPE = (_ph, _pw, N_BANDS) |
| BATCH_SIZE = 8 |
| THRESHOLDS = [0.30, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60, 0.65, 0.70] |
|
|
| os.makedirs(OUT_DIR, exist_ok=True) |
| print(f"[ensemble] {len(CKPT_PATHS)} models | bands={N_BANDS} | patch={_ph}x{_pw}") |
| for p in CKPT_PATHS: |
| print(f" {p}") |
|
|
|
|
| |
| |
| |
| def _load_norm(ckpt_path): |
| cand = os.path.join(os.path.dirname(ckpt_path), "norm_stats.json") |
| if os.path.exists(cand): |
| st = json.load(open(cand)) |
| m = np.array(st["mean"], np.float32).reshape(1, 1, -1) |
| s = np.array(st["std"], np.float32).reshape(1, 1, -1) |
| print(f"[ensemble] norm stats: {cand}") |
| return m, s |
| print("[ensemble] norm stats: built-in fallback") |
| return GLOBAL_MEAN, GLOBAL_STD |
|
|
|
|
| NORM_MEAN, NORM_STD = _load_norm(CKPT_PATHS[0]) |
|
|
|
|
| def metrics_from_cm(tp, fp, fn, tn): |
| prec = tp / (tp + fp + 1e-9) |
| rec = tp / (tp + fn + 1e-9) |
| f1 = 2 * prec * rec / (prec + rec + 1e-9) |
| iou = tp / (tp + fp + fn + 1e-9) |
| dice = 2 * tp / (2 * tp + fp + fn + 1e-9) |
| acc = (tp + tn) / (tp + tn + fp + fn + 1e-9) |
| spec = tn / (tn + fp + 1e-9) |
| return prec, rec, f1, iou, dice, acc, spec |
|
|
|
|
| def predict_tta(model, X, batch_size): |
| """Average sigmoid probs over the 8 dihedral (D4) transforms.""" |
| acc = np.zeros((X.shape[0], X.shape[1], X.shape[2], 1), np.float32) |
| n = 0 |
| for flip in (False, True): |
| for k in (0, 1, 2, 3): |
| Xt = X[:, :, ::-1, :] if flip else X |
| Xt = np.ascontiguousarray(np.rot90(Xt, k, axes=(1, 2))) |
| p = model.predict(Xt, batch_size=batch_size, verbose=0) |
| p = np.rot90(p, -k, axes=(1, 2)) |
| if flip: |
| p = p[:, :, ::-1, :] |
| acc += p |
| n += 1 |
| return acc / n |
|
|
|
|
| |
| |
| |
| feat_files = sorted(glob.glob(os.path.join(TEST_FEAT, '*.npy'))) |
| mask_files = [os.path.join(TEST_MASK, os.path.basename(f)) for f in feat_files] |
| for mf in mask_files: |
| assert os.path.exists(mf), f"Missing mask: {mf}" |
| print(f"\nLoading {len(feat_files)} test patches…") |
|
|
| X, Y, ids = [], [], [] |
| for fp, mp in zip(feat_files, mask_files): |
| feat = np.load(fp).astype(np.float32) |
| mask = np.load(mp).astype(np.float32) |
| feat = np.where((feat == -9999) | ~np.isfinite(feat), 0, feat) |
| if BAND_INDICES is not None: |
| feat = feat[..., BAND_INDICES] |
| feat = (feat - NORM_MEAN) / (NORM_STD + 1e-7) |
| X.append(feat); Y.append(mask) |
| ids.append(os.path.basename(fp).replace('.npy', '')) |
|
|
| X = np.array(X, dtype=np.float32) |
| Y = np.array(Y, dtype=np.float32) |
|
|
| |
| |
| |
| model = build_model(input_shape=INPUT_SHAPE, num_classes=1, num_heads=8) |
|
|
| Y_prob_sum = np.zeros((X.shape[0], X.shape[1], X.shape[2], 1), np.float32) |
| for mi, ckpt in enumerate(CKPT_PATHS, 1): |
| if not os.path.exists(ckpt): |
| raise FileNotFoundError(f"Checkpoint not found: {ckpt}") |
| model.load_weights(ckpt) |
| print(f"[{mi}/{len(CKPT_PATHS)}] TTA predicting with {ckpt} …") |
| Y_prob_sum += predict_tta(model, X, BATCH_SIZE) |
|
|
| Y_prob = Y_prob_sum / len(CKPT_PATHS) |
|
|
| |
| |
| |
| all_true, all_prob, per_image = [], [], [] |
| for i, iid in enumerate(ids): |
| yt = Y[i, :, :, 0].flatten().astype(np.int32) |
| yprob = Y_prob[i, :, :, 0].flatten() |
| all_true.append(yt); all_prob.append(yprob) |
|
|
| yp = (yprob >= 0.5).astype(np.int32) |
| tn, fp, fn, tp = confusion_matrix(yt, yp, labels=[0, 1]).ravel() |
| prec, rec, f1, iou, dice, acc, spec = metrics_from_cm(tp, fp, fn, tn) |
| if (tp + fp + fn) == 0: |
| prec = rec = f1 = iou = dice = 1.0 |
| per_image.append({ |
| 'image_id': iid, |
| 'glacier_frac_gt': round(float(yt.mean()), 4), |
| 'TP': int(tp), 'FP': int(fp), 'FN': int(fn), 'TN': int(tn), |
| 'precision': round(prec, 4), 'recall': round(rec, 4), |
| 'f1_score': round(f1, 4), 'iou': round(iou, 4), 'dice': round(dice, 4), |
| }) |
|
|
| per_image_df = pd.DataFrame(per_image).sort_values('iou').reset_index(drop=True) |
| per_image_df.to_csv(os.path.join(OUT_DIR, 'per_image_metrics.csv'), index=False) |
|
|
| |
| |
| |
| y_true_flat = np.concatenate(all_true) |
| y_prob_flat = np.concatenate(all_prob) |
|
|
| rows = [] |
| for t in THRESHOLDS: |
| yp = (y_prob_flat >= t).astype(np.int32) |
| tn, fp, fn, tp = confusion_matrix(y_true_flat, yp, labels=[0, 1]).ravel() |
| prec, rec, f1, iou, dice, acc, spec = metrics_from_cm(tp, fp, fn, tn) |
| rows.append({'threshold': t, 'precision': round(prec, 6), 'recall': round(rec, 6), |
| 'f1_score': round(f1, 6), 'iou': round(iou, 6), 'dice': round(dice, 6)}) |
| thresh_df = pd.DataFrame(rows) |
| thresh_df.to_csv(os.path.join(OUT_DIR, 'evaluation_metrics.csv'), index=False) |
|
|
| best_row = thresh_df.loc[thresh_df['iou'].idxmax()] |
| best_t = float(best_row['threshold']) |
|
|
| |
| gl = per_image_df[per_image_df['glacier_frac_gt'] > 0.01] |
|
|
| with open(os.path.join(OUT_DIR, 'metrics.json'), 'w') as f: |
| json.dump({ |
| 'n_models': len(CKPT_PATHS), |
| 'best_threshold': best_t, |
| 'test_precision': round(float(best_row['precision']), 4), |
| 'test_recall': round(float(best_row['recall']), 4), |
| 'test_f1': round(float(best_row['f1_score']), 4), |
| 'test_iou': round(float(best_row['iou']), 4), |
| 'test_dice': round(float(best_row['dice']), 4), |
| 'glacier_only_dice': round(float(gl['dice'].mean()), 4), |
| 'glacier_only_iou': round(float(gl['iou'].mean()), 4), |
| }, f, indent=2) |
|
|
| |
| |
| |
| lines = [ |
| "=" * 60, |
| f" ENSEMBLE EVALUATION — {len(CKPT_PATHS)} SwinUNet models", |
| "=" * 60, |
| "", |
| f"Test patches : {len(ids)}", |
| f"Best threshold by IoU : {best_t}", |
| "", |
| "GLOBAL (pixel-pooled) @ best threshold:", |
| f" precision : {best_row['precision']:.4f}", |
| f" recall : {best_row['recall']:.4f}", |
| f" f1 : {best_row['f1_score']:.4f}", |
| f" iou : {best_row['iou']:.4f}", |
| f" dice : {best_row['dice']:.4f}", |
| "", |
| f"GLACIER-ONLY per-image mean ({len(gl)} patches):", |
| f" dice : {gl['dice'].mean():.4f}", |
| f" iou : {gl['iou'].mean():.4f}", |
| "", |
| "Threshold sweep:", |
| " {:<8} {:<10} {:<10} {:<10} {:<10}".format("thresh", "precision", "recall", "iou", "dice"), |
| " " + "-" * 52, |
| ] |
| for _, r in thresh_df.iterrows(): |
| tag = " <- best" if r['threshold'] == best_t else "" |
| lines.append(" {:<8} {:<10.4f} {:<10.4f} {:<10.4f} {:<10.4f}{}".format( |
| r['threshold'], r['precision'], r['recall'], r['iou'], r['dice'], tag)) |
| lines += ["", f"Output dir: {OUT_DIR}"] |
|
|
| summary = "\n".join(lines) |
| print("\n" + summary) |
| with open(os.path.join(OUT_DIR, 'evaluation_summary.txt'), 'w') as f: |
| f.write(summary) |
| print("\nEnsemble evaluation complete.") |
|
|
