Add evaluation configuration and update inference pipeline

forecasting/inference/auto_evaluate.py

@@ -1,451 +0,0 @@
-#!/usr/bin/env python3
-"""
-Automated Evaluation Script for Solar Flare Models
-
-This script automates the generation of inference and evaluation configurations,
-and runs the complete end-to-end evaluation pipeline for trained solar flare models.
-
-It supports both directory-based checkpoint discovery and direct checkpoint paths,
-automatically detecting the model type and setting up inference/evaluation YAMLs.
-
-Usage
------
-Example commands:
-    python auto_evaluate.py -checkpoint_dir /path/to/checkpoint/dir -model_name my_model
-    python auto_evaluate.py -checkpoint_path /path/to/checkpoint.pth -model_name my_model
-"""
-
-import argparse
-from operator import truediv
-import os
-import subprocess
-import sys
-import yaml
-from pathlib import Path
-from datetime import datetime
-import glob
-
-# Add project root to Python path
-PROJECT_ROOT = Path(__file__).parent.parent.parent.absolute()
-sys.path.insert(0, str(PROJECT_ROOT))
-
-
-def find_checkpoint_files(checkpoint_dir):
-    """
-    Find all checkpoint files (.pth, .ckpt, .pt) within a directory.
-
-    Parameters
-    ----------
-    checkpoint_dir : str or Path
-        Path to the directory containing model checkpoint files.
-
-    Returns
-    -------
-    list of str
-        Sorted list of checkpoint file paths discovered within the directory.
-    """
-    patterns = ['*.pth', '*.ckpt', '*.pt']
-    checkpoints = []
-    
-    for pattern in patterns:
-        checkpoints.extend(glob.glob(str(Path(checkpoint_dir) / pattern)))
-        checkpoints.extend(glob.glob(str(Path(checkpoint_dir) / '**' / pattern), recursive=True))
-    
-    return sorted(checkpoints)
-
-
-def detect_model_type(checkpoint_path):
-    """
-    Infer the model type from a checkpoint filename.
-
-    Parameters
-    ----------
-    checkpoint_path : str
-        Path to the checkpoint file.
-
-    Returns
-    -------
-    str
-        Model type inferred from filename (e.g., 'vitlocal', 'vitpatch', 'fusion', etc.).
-    """
-    filename = Path(checkpoint_path).name.lower()
-    
-    return 'vitlocal'
-
-
-def check_sxr_data_availability(base_data_dir):
-    """
-    Check if SXR data is available in the specified directory.
-
-    Parameters
-    ----------
-    base_data_dir : str
-        Base directory containing the SXR data.
-
-    Returns
-    -------
-    bool
-        True if SXR data is available, False otherwise.
-    """
-    sxr_dir = Path(base_data_dir) / "SXR"
-    sxr_norm_path = Path(base_data_dir) / "SXR" / "normalized_sxr.npy"
-    
-    # Check if SXR directory exists and has files
-    if not sxr_dir.exists():
-        print(f"SXR directory not found: {sxr_dir}")
-        return False
-    
-    # Check if normalized SXR file exists
-    if not sxr_norm_path.exists():
-        print(f"Normalized SXR file not found: {sxr_norm_path}")
-        return False
-    
-    # Check if there are any .npy files in the SXR directory
-    sxr_files = list(sxr_dir.glob("*.npy"))
-    if not sxr_files:
-        print(f"No SXR data files found in: {sxr_dir}")
-        return False
-    
-    print(f"Found {len(sxr_files)} SXR data files in {sxr_dir}")
-    return True
-
-
-def create_inference_config(checkpoint_path, model_name, base_data_dir="/mnt/data/NO-OVERLAP", prediction_only=False):
-    """
-    Dynamically create an inference configuration dictionary for a given checkpoint.
-
-    Parameters
-    ----------
-    checkpoint_path : str
-        Path to the checkpoint file.
-    model_name : str
-        Name for the model (used for output folder and file naming).
-    base_data_dir : str, optional
-        Root directory of dataset and normalization files.
-    prediction_only : bool, optional
-        If True, run in prediction-only mode (no SXR ground truth required).
-    Returns
-    -------
-    tuple(dict, str)
-        - Inference configuration dictionary.
-        - Path to the output directory where results will be saved.
-    """
-    # Detect model type
-    model_type = detect_model_type(checkpoint_path)
-    
-    # Create output directory
-    output_dir = f"/Volumes/T9/FOXES_Data/paper_res/{model_name}"
-    os.makedirs(output_dir, exist_ok=True)
-    os.makedirs(f"{output_dir}/weights", exist_ok=True)
-    
-    # Create flux directory for patch-based models
-    if model_type == 'vitlocal':
-        os.makedirs(f"{output_dir}/flux", exist_ok=True)
-    
-    # Generate config
-    config = {
-        'SolO': 'false',
-        'Stereo': 'false',
-        'prediction_only': 'true' if prediction_only else 'false',
-        'base_data_dir': base_data_dir,
-        'data': {
-            'aia_dir': f"{base_data_dir}/AIA/",
-            'checkpoint_path': checkpoint_path,
-            'sxr_dir': f"{base_data_dir}/SXR/" if not prediction_only else "",
-            'sxr_norm_path': f"{base_data_dir}/SXR/normalized_sxr.npy" if not prediction_only else ""
-        },
-        'model': model_type,
-        'wavelengths': [94, 131, 171, 193, 211, 304, 335],
-        'mc': {
-            'active': 'false',
-            'runs': 5
-        },
-        'model_params': {
-            'batch_size': 8,  # Match training batch size. If you get OOM errors, reduce this.
-                              # Note: Inference with attention weights uses more memory than training
-            'input_size': 512,
-            'no_weights': True,  # Set to False to save attention weights (uses more memory)
-            'no_flux': False,  # Set to False to save flux contributions (uses more memory)
-            'patch_size': 8
-        },
-        'vit_architecture': {
-            'embed_dim': 256,
-            'hidden_dim': 1024,
-            'num_channels': 7,
-            'num_classes': 1,
-            'patch_size': 16,
-            'num_patches': 1024,
-            'num_heads': 8,
-            'num_layers': 8,
-            'dropout': 0.1
-        },
-        'output_path': f"{output_dir}/{model_name}_predictions.csv",
-        'weight_path': f"{output_dir}/weights"
-    }
-    
-    # Add flux_path for patch-based models
-    if model_type in ['vitpatch', 'vitlocal']:
-        config['flux_path'] = f"{output_dir}/flux/"
-    
-    
-    return config, output_dir
-
-
-def create_evaluation_config(model_name, output_dir, base_data_dir="/mnt/data/NO-OVERLAP",
-                             prediction_only=False, regression_background='black'):
-    """
-    Create evaluation configuration for computing metrics and visualizations.
-
-    Parameters
-    ----------
-    model_name : str
-        Name of the model under evaluation.
-    output_dir : str
-        Path to output directory containing prediction results.
-    base_data_dir : str, optional
-        Base dataset directory containing AIA and SXR test data.
-    prediction_only : bool, optional
-        If True, create config for prediction-only mode (no ground truth evaluation).
-
-    Returns
-    -------
-    dict
-        Evaluation configuration dictionary with metrics, time range, and plotting settings.
-    """
-    config = {
-        'base_data_dir': base_data_dir,
-        'output_base_dir': f"{base_data_dir}/solar_flare_comparison_results",
-        'prediction_only': prediction_only,
-        'data': {
-            'aia_dir': f"{base_data_dir}/AIA/test/",
-            'weight_path': f"{output_dir}/weights"
-        },
-        'model_predictions': {
-            'main_model_csv': f"{output_dir}/{model_name}_predictions.csv",
-            'baseline_csv': ''
-        },
-        'evaluation': {
-            'output_dir': output_dir,
-            'sxr_cutoff': 1e-10 if not prediction_only else None
-        },
-        'time_range': {
-            'start_time': '2023-08-05T21:00:00',
-            'end_time': '2023-08-05T23:59:00',
-            'interval_minutes': 5
-        },
-        'plotting': {
-            'figure_size': [12, 8],
-            'dpi': 300,
-            'colormap': 'sdoaia171',
-            'regression_background': regression_background
-        },
-        'metrics': {
-            'include_rmse': True,
-            'include_mae': True,
-            'include_r2': True,
-            'include_correlation': True
-        }
-    }
-    return config
-
-
-def run_inference(inference_config_path):
-    """
-    Execute model inference using the generated YAML configuration.
-
-    Parameters
-    ----------
-    inference_config_path : str
-        Path to the inference configuration YAML file.
-
-    Returns
-    -------
-    bool
-        True if inference completes successfully, False if an error occurs.
-    """
-    print(f"Running inference with config: {inference_config_path}")
-    
-    cmd = [
-        sys.executable, 
-        str(PROJECT_ROOT / "forecasting/inference/inference.py"),
-        "-config", inference_config_path
-    ]
-    
-    # Use Popen with real-time output streaming to show progress bar
-    # Both stdout and stderr go to terminal so tqdm progress bar (which writes to stderr) is visible
-    process = subprocess.Popen(
-        cmd,
-        stdout=None,  # Let stdout go directly to terminal
-        stderr=subprocess.STDOUT,  # Merge stderr into stdout so progress bar is visible
-        text=True,
-        bufsize=1  # Line buffered for real-time output
-    )
-    
-    # Wait for process to complete
-    process.wait()
-    
-    if process.returncode != 0:
-        print(f"Error: Inference process exited with code {process.returncode}")
-        return False
-    
-    print("Inference completed successfully!")
-    return True
-
-
-def run_evaluation(evaluation_config_path):
-    """
-    Execute evaluation of inference outputs using the generated YAML configuration.
-
-    Parameters
-    ----------
-    evaluation_config_path : str
-        Path to the evaluation configuration YAML file.
-
-    Returns
-    -------
-    bool
-        True if evaluation completes successfully, False otherwise.
-    """
-    print(f"Running evaluation with config: {evaluation_config_path}")
-    
-    cmd = [
-        sys.executable, 
-        str(PROJECT_ROOT / "forecasting/inference/evaluation.py"),
-        "-config", evaluation_config_path
-    ]
-    
-    # Use Popen with real-time output streaming
-    # Both stdout and stderr go to terminal for real-time output
-    process = subprocess.Popen(
-        cmd,
-        stdout=None,  # Let stdout go directly to terminal
-        stderr=subprocess.STDOUT,  # Merge stderr into stdout
-        text=True,
-        bufsize=1  # Line buffered for real-time output
-    )
-    
-    # Wait for process to complete
-    process.wait()
-    
-    if process.returncode != 0:
-        print(f"Error: Evaluation process exited with code {process.returncode}")
-        return False
-    
-    print("Evaluation completed successfully!")
-    return True
-
-
-def main():
-    """
-    Main function for automating inference and evaluation.
-
-    Steps:
-      1. Parse command-line arguments.
-      2. Locate checkpoint file or directory.
-      3. Generate inference and evaluation YAML configs.
-      4. Optionally run inference and/or evaluation scripts.
-      5. Output results and metrics to specified directory.
-    """
-    parser = argparse.ArgumentParser(description='Automated evaluation for solar flare models')
-    parser.add_argument('-checkpoint_dir', type=str, help='Directory containing checkpoint files')
-    parser.add_argument('-checkpoint_path', type=str, help='Specific checkpoint file path')
-    parser.add_argument('-model_name', type=str, required=True, help='Name for the model (used for output naming)')
-    parser.add_argument('-base_data_dir', type=str, default='/data/FOXES_Data/', help='Base data directory')
-    parser.add_argument('-skip_inference', action='store_true', help='Skip inference and only run evaluation')
-    parser.add_argument('-skip_evaluation', action='store_true', help='Skip evaluation and only run inference')
-    parser.add_argument('-prediction_only', action='store_true', help='Force prediction-only mode (no SXR ground truth)')
-    parser.add_argument('-regression_background', type=str, choices=['black', 'white'], default='black',
-                        help='Background color for regression plots (default: black)')
-    
-    args = parser.parse_args()
-    
-    # Determine checkpoint path
-    if args.checkpoint_path:
-        checkpoint_path = args.checkpoint_path
-        if not os.path.exists(checkpoint_path):
-            print(f"Error: Checkpoint file not found: {checkpoint_path}")
-            sys.exit(1)
-    elif args.checkpoint_dir:
-        checkpoints = find_checkpoint_files(args.checkpoint_dir)
-        if not checkpoints:
-            print(f"Error: No checkpoint files found in {args.checkpoint_dir}")
-            sys.exit(1)
-        elif len(checkpoints) > 1:
-            print(f"Found multiple checkpoints: {checkpoints}")
-            print("Using the first one. Use -checkpoint_path to specify a specific file.")
-        checkpoint_path = checkpoints[0]
-    else:
-        print("Error: Must specify either -checkpoint_dir or -checkpoint_path")
-        sys.exit(1)
-    
-    print(f"Using checkpoint: {checkpoint_path}")
-    print(f"Model name: {args.model_name}")
-    
-    # Check SXR data availability and determine if we should use prediction-only mode
-    prediction_only_mode = args.prediction_only
-    
-    if not prediction_only_mode:
-        print("Checking SXR data availability...")
-        sxr_available = check_sxr_data_availability(args.base_data_dir)
-        if not sxr_available:
-            print("⚠️  SXR data not available. Switching to prediction-only mode.")
-            prediction_only_mode = True
-        else:
-            print("✅ SXR data found. Running with ground truth evaluation.")
-    else:
-        print("🔮 Running in prediction-only mode (as requested).")
-    
-    # Create configs
-    inference_config, output_dir = create_inference_config(checkpoint_path, args.model_name, args.base_data_dir, prediction_only_mode)
-    evaluation_config = create_evaluation_config(
-        args.model_name,
-        output_dir,
-        args.base_data_dir,
-        prediction_only_mode,
-        regression_background=args.regression_background
-    )
-    
-    # Save configs
-    inference_config_path = f"/tmp/inference_config_{args.model_name}.yaml"
-    evaluation_config_path = f"/tmp/evaluation_config_{args.model_name}.yaml"
-    
-    with open(inference_config_path, 'w') as f:
-        yaml.dump(inference_config, f, default_flow_style=False)
-    
-    with open(evaluation_config_path, 'w') as f:
-        yaml.dump(evaluation_config, f, default_flow_style=False)
-    
-    print(f"Configs saved to:")
-    print(f"  Inference: {inference_config_path}")
-    print(f"  Evaluation: {evaluation_config_path}")
-    print(f"  Output directory: {output_dir}")
-    
-    # Run inference
-    if not args.skip_inference:
-        if not run_inference(inference_config_path):
-            print("Inference failed. Stopping.")
-            sys.exit(1)
-    else:
-        print("Skipping inference...")
-    
-    # Run evaluation
-    if not args.skip_evaluation:
-        if prediction_only_mode:
-            print("Skipping evaluation (prediction-only mode - no ground truth available)")
-        else:
-            if not run_evaluation(evaluation_config_path):
-                print("Evaluation failed. Stopping.")
-                sys.exit(1)
-    else:
-        print("Skipping evaluation...")
-    
-    print(f"\n✅ Complete! Results saved to: {output_dir}")
-    if prediction_only_mode:
-        print(f"🔮 Prediction-only mode: No ground truth evaluation performed")
-        print(f"📊 Check the prediction results in: {output_dir}")
-    else:
-        print(f"📊 Check the plots and metrics in: {output_dir}")
-
-
-if __name__ == '__main__':
-    main()

forecasting/inference/evaluation.py

@@ -900,7 +900,7 @@ class SolarFlareEvaluator:
 
                 sxr_ax.set_xlim([pd.to_datetime(timestamp) - pd.Timedelta(hours=4),
                                  pd.to_datetime(timestamp) + pd.Timedelta(hours=4)])
-                sxr_ax.set_ylim([5e-7, 5e-4])  # Set y-limits for SXR data
+                #sxr_ax.set_ylim([5e-7, 5e-4])  # Set y-limits for SXR data
                 sxr_ax.set_ylabel(r'SXR Flux (W/m$^2$)', fontsize=12, fontfamily='Barlow',
                                   color=('white' if is_dark else 'black'))
                 sxr_ax.set_xlabel('Time', fontsize=12, fontfamily='Barlow', color=('white' if is_dark else 'black'))

forecasting/inference/evaluation_config.yaml

@@ -0,0 +1,27 @@
+# =============================================================================
+# FOXES Evaluation Configuration
+# =============================================================================
+# Used by evaluation.py to compute metrics and generate plots.
+#
+# Usage: python evaluation.py -config evaluation_config.yaml
+# =============================================================================
+
+model_predictions:
+  main_model_csv: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
+  baseline_csv:   null   # path to baseline predictions CSV, or null to skip comparison
+
+data:
+  aia_dir:     "/Volumes/T9/Data_FOXES/AIA_processed/val"
+  weight_path: "/Volumes/T9/Data_FOXES/inference/weights"
+
+evaluation:
+  output_dir:  "/Volumes/T9/Data_FOXES/inference/evaluation"
+  sxr_cutoff:  null   # minimum ground-truth SXR value to include; null = no filter
+
+time_range:
+  start_time:       "2023-01-01T00:00:00"
+  end_time:         "2023-12-31T23:59:59"
+  interval_minutes: 60
+
+plotting:
+  regression_background: "black"

forecasting/inference/inference.py

@@ -209,11 +209,10 @@ def evaluate_model_on_dataset(model, dataset, batch_size=16, times=None, config_
                 del flux_contributions
                 flux_contributions = None
             
-            # Force garbage collection and clear GPU cache after EVERY batch
-            # This is critical - memory accumulates between batches otherwise
-            gc.collect()  # Force Python garbage collection
-            torch.cuda.empty_cache()  # Clear PyTorch's GPU cache
-            torch.cuda.synchronize()  # Wait for all operations to complete before clearing
+            gc.collect()
+            if torch.cuda.is_available():
+                torch.cuda.empty_cache()
+                torch.cuda.synchronize()
 
 
 def save_batch_flux_contributions(batch_flux_contributions, batch_idx, batch_size, times, flux_path, sxr_norm=None):
@@ -335,7 +334,7 @@ def load_model_from_config(config_data):
     if ".ckpt" in checkpoint_path:
         # Lightning checkpoint format
         if model_type.lower() == 'vitlocal':
-            model = ViTLocal.load_from_checkpoint(checkpoint_path, map_location=load_device)
+            model = ViTLocal.load_from_checkpoint(checkpoint_path, map_location=load_device, weights_only=False)
         else:
             try:
                 model_class = getattr(models, model_type)
@@ -418,7 +417,7 @@ def main():
         print("  Note: This saves ~3GB per batch by not computing attention weights.")
     else:
         print("Will save attention weights during inference.")
-        print("\n💡 Memory note:")
+        print("\n Memory note:")
         print("   - Attention weights from all layers use significant GPU memory")
         print("   - For ViT with 8 layers, 8 heads, 4096 patches: ~3GB+ per batch with attention!")
         print("   - If you get OOM errors, set no_weights=true to skip attention saving\n")

forecasting/inference/local_config.yaml

@@ -1,21 +1,49 @@
 # =============================================================================
-# Flare Analysis Configuration
+# FOXES Inference + Flare Analysis Configuration
 # =============================================================================
-# Unified config for FOXES flare detection, tracking, and HEK catalog matching
+# Shared config for inference.py and flare_analysis.py
 #
-# Usage: python flare_analysis.py --config flare_analysis_config.yaml
+# Usage:
+#   python inference.py      -config local_config.yaml
+#   python flare_analysis.py --config local_config.yaml
 # =============================================================================
 
 # -----------------------------------------------------------------------------
-# Data Paths
+# Inference (inference.py)
+# -----------------------------------------------------------------------------
+model: "ViTLocal"
+wavelengths: [94, 131, 171, 193, 211, 304, 335]
+SolO: "false"
+Stereo: "false"
+prediction_only: "false"
+
+data:
+  aia_dir:         "/Volumes/T9/Data_FOXES/AIA_processed"
+  sxr_dir:         "/Volumes/T9/Data_FOXES/SXR_processed"
+  sxr_norm_path:   "/Volumes/T9/Data_FOXES/SXR_processed/normalized_sxr.npy"
+  checkpoint_path: "/Volumes/T9/Data_FOXES/checkpoints/best.ckpt"  # update to actual checkpoint
+
+output_path: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
+weight_path: "/Volumes/T9/Data_FOXES/inference/weights/"
+flux_path:   "/Volumes/T9/Data_FOXES/inference/flux/"
+
+model_params:
+  input_size: 512
+  patch_size: 8
+  batch_size: 10
+  no_weights: false
+  no_flux:    false
+
+# -----------------------------------------------------------------------------
+# Flare Analysis (flare_analysis.py)
 # -----------------------------------------------------------------------------
 paths:
-  data_dir: "/Volumes/T9/FOXES_Data"
-  flux_path: "/Volumes/T9/batch_results/vit/flux"
-  aia_path: "/Volumes/T9/FOXES_Data/AIA"
-  predictions_csv: "/Volumes/T9/batch_results/vit/vit_predictions_all.csv"
-  hek_catalog: null # null to auto-fetch
-  output_dir: "/Volumes/T9/flux_output"  # Directory for plots and movies
+  data_dir:        "/Volumes/T9/Data_FOXES"
+  flux_path:       "/Volumes/T9/Data_FOXES/inference/flux"
+  aia_path:        "/Volumes/T9/Data_FOXES/AIA_processed/val"
+  predictions_csv: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
+  hek_catalog: null
+  output_dir:      "/Volumes/T9/Data_FOXES/inference/output"
 
 # -----------------------------------------------------------------------------
 # Time Range

forecasting/models/vit_patch_model_local.py

@@ -23,10 +23,11 @@ class ViTLocal(pl.LightningModule):
     def __init__(self, model_kwargs, sxr_norm, base_weights=None):
         super().__init__()
         self.model_kwargs = model_kwargs
-        self.lr = model_kwargs['learning_rate']
+        self.lr = model_kwargs.get('learning_rate', model_kwargs.get('lr', 1e-4))
         self.save_hyperparameters()
         filtered_kwargs = dict(model_kwargs)
         filtered_kwargs.pop('learning_rate', None)
+        filtered_kwargs.pop('lr', None)
         filtered_kwargs.pop('num_classes', None)
         self.model = VisionTransformerLocal(**filtered_kwargs)
         self.base_weights = base_weights

pipeline_config.yaml

@@ -74,7 +74,6 @@ train:
         - aia
         - sxr
         - regression
-      run_name: paper-8-patch-4ch
       notes: Regression from AIA images to SXR images using ViTLocal model with 8x8 patches
 
 # -----------------------------------------------------------------------------
@@ -83,5 +82,31 @@ train:
 inference:
   config: "forecasting/inference/local_config.yaml"
   overrides:                          # Any key from local_config.yaml can go here
+    data:
+      aia_dir:         "/Volumes/T9/Data_FOXES/AIA_processed"
+      sxr_dir:         "/Volumes/T9/Data_FOXES/SXR_processed"
+      sxr_norm_path:   "/Volumes/T9/Data_FOXES/SXR_processed/normalized_sxr.npy"
+      checkpoint_path: "/Volumes/T9/FOXES_Misc/final_checkpoint/paper-8-patch-weights-epoch=100-val_total_loss=0.0048.ckpt"  # update to actual checkpoint
+    output_path: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
     paths:
-      data_dir: "/Volumes/T9/Data_FOXES"
+      data_dir:        "/Volumes/T9/Data_FOXES"
+      predictions_csv: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
+      aia_path:        "/Volumes/T9/Data_FOXES/AIA_processed/val"
+
+# -----------------------------------------------------------------------------
+# Evaluation  (step: evaluate)
+# -----------------------------------------------------------------------------
+evaluate:
+  config: "forecasting/inference/evaluation_config.yaml"
+  overrides:                          # Any key from evaluation_config.yaml can go here
+    model_predictions:
+      main_model_csv: "/Volumes/T9/Data_FOXES/inference/predictions.csv"
+    data:
+      aia_dir:     "/Volumes/T9/Data_FOXES/AIA_processed/val"
+      weight_path: "/Volumes/T9/Data_FOXES/inference/weights"
+    evaluation:
+      output_dir:  "/Volumes/T9/Data_FOXES/inference/evaluation"
+    time_range:
+      start_time:       "2023-01-01T00:00:00"
+      end_time:         "2023-12-31T23:59:59"
+      interval_minutes: 60

run_pipeline.py

@@ -84,6 +84,7 @@ STEP_ORDER = [
     "normalize",
     "train",
     "inference",
+    "evaluate",
     "flare_analysis",
 ]
 
@@ -120,6 +121,10 @@ STEP_INFO = {
         "description": "Run batch inference and save predictions CSV",
         "script": ROOT / "forecasting" / "inference" / "inference.py",
     },
+    "evaluate": {
+        "description": "Compute metrics and generate evaluation plots from predictions CSV",
+        "script": ROOT / "forecasting" / "inference" / "evaluation.py",
+    },
     "flare_analysis": {
         "description": "Detect, track, and match flares; generate plots/movies",
         "script": ROOT / "forecasting" / "inference" / "flare_analysis.py",
@@ -234,6 +239,15 @@ def build_commands(step: str, cfg: dict, force: bool) -> list[list[str]] | None:
             config_path = str(write_merged_config(config_path, inf["overrides"], "inference_config"))
         return [base + ["-config", config_path]]
 
+    if step == "evaluate":
+        if not require(["config"], "evaluate"):
+            return None
+        ev = cfg["evaluate"]
+        config_path = ev["config"]
+        if ev.get("overrides"):
+            config_path = str(write_merged_config(config_path, ev["overrides"], "evaluate_config"))
+        return [base + ["-config", config_path]]
+
     if step == "flare_analysis":
         if not require(["config"], "inference"):
             return None