ml/scripts/sanity_check.py

"""
3-epoch sanity check per crop (item 10).

For each crop:
  * load data (triggers folder-map verify, corrupt-skip, split-dist, coupling asserts)
  * train 3 epochs with a frozen backbone
  * predict on the full validation set
  * print confusion matrix + per-class accuracy
  * detect mode-collapse (predictions concentrated on a single class)

Returns a structured result so the orchestrator can SKIP collapsed crops instead
of wasting a 50-epoch run on them.

Usage:
    python -m ml.scripts.sanity_check                 # all crops
    python -m ml.scripts.sanity_check --crop soybean  # one crop
"""
import argparse
import os
import sys
from pathlib import Path

os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")

ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT))

import numpy as np  # noqa: E402
from sklearn.metrics import confusion_matrix  # noqa: E402

from ml.config import CROPS  # noqa: E402
from ml.utils.data_loader import CropDatasetLoader  # noqa: E402
from ml.utils.model_builder import build_model  # noqa: E402

# Predictions are considered "collapsed" if a single predicted class accounts for
# more than this fraction of the validation set.
COLLAPSE_THRESHOLD = 0.90


def _print_confusion_matrix(cm: np.ndarray, class_names: list) -> float:
    col_w = max(14, max(len(n) for n in class_names) + 2)
    corner = "true\\pred"
    header = f"{corner:>{col_w}}" + "".join(f"{n[:col_w-1]:>{col_w}}" for n in class_names)
    print("\n" + "=" * len(header))
    print("CONFUSION MATRIX  (rows = true class, cols = predicted class)")
    print("=" * len(header))
    print(header)
    for i, row in enumerate(cm):
        lbl = class_names[i][: col_w - 1]
        print(f"{lbl:>{col_w}}" + "".join(f"{v:>{col_w}}" for v in row))
    print("\nPer-class recall:")
    for i, name in enumerate(class_names):
        total = cm[i].sum()
        correct = cm[i, i]
        pct = correct / total if total > 0 else 0.0
        print(f"  {name:<30}  {correct:>4}/{total:<4}  =  {pct:.1%}")
    overall = cm.diagonal().sum() / cm.sum() if cm.sum() else 0.0
    print(f"\nOverall val accuracy: {overall:.4f}  ({cm.diagonal().sum()}/{cm.sum()})")
    return overall


def sanity_check_crop(crop: str, epochs: int = 3) -> dict:
    """Run a short sanity check for one crop. Returns a result dict."""
    print(f"\n{'#'*70}")
    print(f"# SANITY CHECK — {crop.upper()}  ({epochs} epochs, frozen backbone)")
    print(f"{'#'*70}\n")

    result = {
        "crop": crop, "status": "unknown", "val_accuracy": None,
        "collapsed": None, "dominant_pred_share": None, "n_pred_classes": None,
        "class_names": None, "error": None,
    }
    try:
        loader = CropDatasetLoader(crop)
        images, labels, class_names = loader.load_dataset()
        result["class_names"] = class_names

        train_gen, val_gen, y_train = loader.create_data_generators(images, labels)

        num_classes = len(class_names)
        print(f"\nBuilding EfficientNetB0 model ({num_classes} classes) ...")
        model = build_model(num_classes=num_classes, crop=crop, architecture="EfficientNetB0")

        print(f"\nTraining {epochs} epochs (frozen backbone) ...")
        model.fit(train_gen, epochs=epochs, validation_data=val_gen, verbose=2)

        # Predict on the full validation set
        X_val, y_val_cat = val_gen.x, val_gen.y
        y_val = np.argmax(y_val_cat, axis=1)
        y_pred = np.argmax(model.predict(X_val, verbose=0, batch_size=32), axis=1)

        cm = confusion_matrix(y_val, y_pred, labels=list(range(num_classes)))
        overall = _print_confusion_matrix(cm, class_names)

        # Collapse detection
        pred_unique, pred_counts = np.unique(y_pred, return_counts=True)
        dominant_share = float(pred_counts.max() / pred_counts.sum())
        n_pred_classes = int(len(pred_unique))
        collapsed = (n_pred_classes == 1) or (dominant_share > COLLAPSE_THRESHOLD)

        result.update({
            "val_accuracy": float(overall),
            "collapsed": collapsed,
            "dominant_pred_share": dominant_share,
            "n_pred_classes": n_pred_classes,
        })

        print(f"\nPredicted-class spread: {n_pred_classes}/{num_classes} classes used, "
              f"dominant class = {dominant_share:.1%} of predictions")
        if collapsed:
            result["status"] = "COLLAPSED"
            print("✗  MODE COLLAPSE detected — model predicts one class for "
                  f"{dominant_share:.0%} of val. This crop would waste a full run.")
        elif overall >= 0.45:
            result["status"] = "PASS"
            print("✓  PASS — clean label spread, no collapse, reasonable accuracy.")
        else:
            result["status"] = "WEAK"
            print("⚠  WEAK — no collapse but accuracy < 45% at epoch 3 (still trainable).")
    except Exception as e:
        import traceback
        result["status"] = "ERROR"
        result["error"] = str(e)
        print(f"✗  ERROR during sanity check for {crop}: {e}")
        traceback.print_exc()
    return result


def main(crop: str = None, epochs: int = 3) -> int:
    crops = [crop] if crop else list(CROPS.keys())
    results = [sanity_check_crop(c, epochs=epochs) for c in crops]

    print(f"\n\n{'='*70}")
    print("SANITY CHECK SUMMARY")
    print(f"{'='*70}")
    print(f"{'crop':<10}{'status':<12}{'val_acc':<10}{'pred_classes':<14}{'dom_share':<10}")
    for r in results:
        va = f"{r['val_accuracy']:.3f}" if r["val_accuracy"] is not None else "—"
        npc = f"{r['n_pred_classes']}" if r["n_pred_classes"] is not None else "—"
        ds = f"{r['dominant_pred_share']:.0%}" if r["dominant_pred_share"] is not None else "—"
        print(f"{r['crop']:<10}{r['status']:<12}{va:<10}{npc:<14}{ds:<10}")
    print()
    return 0


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--crop", default=None, choices=list(CROPS.keys()),
                        help="Crop to sanity-check (default: all crops)")
    parser.add_argument("--epochs", type=int, default=3)
    args = parser.parse_args()
    sys.exit(main(args.crop, args.epochs))