import torch import torch.nn as nn from PIL import Image import os import numpy as np from typing import Tuple, List, Optional import logging # Import the model architecture used for prediction; it must match the checkpoint. from proposed_uncertainty import UNet from scipy import io import matplotlib.pyplot as plt from scipy.ndimage import zoom import cv2 import time class Predictor: """ Predictor for M and D matrices. """ def __init__(self, model_path: str, crop_size=None, device: Optional[str] = None): """ Initialize the predictor. Args: model_path: Path to the model weights. crop_size: Crop dimensions. device: Device to use ('cuda' or 'cpu'); select automatically when None. """ # Configure logging. self._setup_logger() # Configure the device. self.device = self._setup_device(device) self.logger.info(f"Using device: {self.device}") # Process the crop dimensions. if crop_size is not None: if isinstance(crop_size, (int, float)): self.crop_height = self.crop_width = int(crop_size) elif isinstance(crop_size, (tuple, list)) and len(crop_size) == 2: self.crop_height, self.crop_width = crop_size else: raise ValueError("crop_size should be an integer or a tuple of (height, width)") self.logger.info(f"Using crop size: ({self.crop_height}, {self.crop_width})") else: self.crop_height = self.crop_width = None self.logger.info("Cropping disabled") # Load the model. try: self.model = self._load_model(model_path) self.logger.info("Model loaded successfully") except Exception as e: self.logger.error(f"Failed to load model: {str(e)}") raise def _setup_logger(self): """Configure logging.""" self.logger = logging.getLogger('Predictor') self.logger.setLevel(logging.INFO) if not self.logger.handlers: handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) self.logger.addHandler(handler) def _setup_device(self, device: Optional[str]) -> torch.device: """Configure the compute device.""" if device is None: device = 'cuda' if torch.cuda.is_available() else 'cpu' return torch.device(device) def _load_model(self, model_path: str) -> nn.Module: """Load the model.""" if not os.path.exists(model_path): raise FileNotFoundError(f"Model file not found: {model_path}") # Create the model. model = UNet(n_channels=3, bilinear=False) checkpoint = torch.load(model_path, map_location=self.device) model.load_state_dict(checkpoint['model_state_dict']) model = model.to(self.device) model.eval() return model def _process_images(self, img1_path: str, img2_path: str, img3_path: str) -> Tuple[torch.Tensor, Tuple[int, int]]: """ Convert three images into network input. Returns the processed tensor and original dimensions: (tensor, (orig_height, orig_width)). """ # Read the three images. img1 = np.array(Image.open(img1_path), dtype=np.float32) img2 = np.array(Image.open(img2_path), dtype=np.float32) img3 = np.array(Image.open(img3_path), dtype=np.float32) # Preserve the original dimensions. orig_height, orig_width = img1.shape # Calculate sina and cosa. S = 3 sina = np.zeros_like(img1, dtype=np.float32) cosa = np.zeros_like(img1, dtype=np.float32) for n in range(1, S + 1): sina -= [img1, img2, img3][n - 1] * np.sin(2 * np.pi * (n - 1) / S) cosa += [img1, img2, img3][n - 1] * np.cos(2 * np.pi * (n - 1) / S) # Normalize by dividing directly by 255. # sina = sina / 1000 # cosa = cosa / 1000 img1 = img1 / 1000 img2 = img2 / 1000 img3 = img3 / 1000 # Crop when requested. if self.crop_height is not None and self.crop_width is not None: H, W = img1.shape start_h = (H - self.crop_height) // 2 start_w = (W - self.crop_width) // 2 img1 = img1[start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] img2 = img2[start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] img3 = img3[start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] img1_tensor = torch.from_numpy(img1).float().unsqueeze(0) # [1, H, W] img2_tensor = torch.from_numpy(img2).float().unsqueeze(0) # [1, H, W] img3_tensor = torch.from_numpy(img3).float().unsqueeze(0) # [1, H, W] input_tensor = torch.cat([img1_tensor, img2_tensor, img3_tensor], dim=0) return input_tensor, (orig_height, orig_width) def create_mask(self, img1_path: str, img2_path: str, img3_path: str) -> Tuple[np.ndarray, Tuple[int, int]]: """ Create a mask that is zero where all three images have zero-valued pixels. Returns the mask and original dimensions: (mask, (orig_height, orig_width)). """ img1 = np.array(Image.open(img1_path)) img2 = np.array(Image.open(img2_path)) img3 = np.array(Image.open(img3_path)) # Preserve the original dimensions. orig_height, orig_width = img1.shape # Create the mask. mask = np.ones_like(img1, dtype=np.float32) mask[(img1 == 0) & (img2 == 0) & (img3 == 0)] = 0 # Crop when requested. if self.crop_height is not None and self.crop_width is not None: H, W = mask.shape start_h = (H - self.crop_height) // 2 start_w = (W - self.crop_width) // 2 mask = mask[start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] return mask, (orig_height, orig_width) def _restore_to_original_size(self, data: np.ndarray, orig_size: Tuple[int, int]) -> np.ndarray: """ Restore cropped data to the center of the original dimensions. Args: data: Cropped data. orig_size: Original dimensions as (height, width). Returns: Data restored to the original dimensions. """ if self.crop_height is None or self.crop_width is None: return data orig_height, orig_width = orig_size result = np.zeros((orig_height, orig_width), dtype=data.dtype) # Calculate the placement coordinates. start_h = (orig_height - self.crop_height) // 2 start_w = (orig_width - self.crop_width) // 2 # Place the cropped data in the center. result[start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] = data return result def predict_single(self, img1_path: str, img2_path: str, img3_path: str) -> Tuple[np.ndarray, np.ndarray]: """ Predict the M and D matrices once. """ self.logger.info("Starting single prediction...") try: # Process the input images. inputs, orig_size = self._process_images(img1_path, img2_path, img3_path) mask, _ = self.create_mask(img1_path, img2_path, img3_path) mask = torch.from_numpy(mask).unsqueeze(0) # [1, H, W] # Use evaluation mode during prediction to disable dropout. self.model.eval() with torch.no_grad(): inputs = inputs.unsqueeze(0).to(self.device) # [1, 3, H, W] outputs= self.model(inputs) # Process the outputs. M = outputs['M_mean'].squeeze().cpu().numpy() # [H, W] D = outputs['D_mean'].squeeze().cpu().numpy() # [H, W] # Apply the mask. M *= mask.squeeze().numpy() D *= mask.squeeze().numpy() # Restore the original dimensions. M = self._restore_to_original_size(M, orig_size) D = self._restore_to_original_size(D, orig_size) # Create a mask with the original dimensions. full_mask, _ = self.create_mask(img1_path, img2_path, img3_path) if self.crop_height is not None and self.crop_width is not None: # Reconstruct the full mask. orig_height, orig_width = orig_size full_mask = np.ones((orig_height, orig_width), dtype=np.float32) full_mask[(np.array(Image.open(img1_path)) == 0) & (np.array(Image.open(img2_path)) == 0) & (np.array(Image.open(img3_path)) == 0)] = 0 # Apply the full mask. M *= full_mask D *= full_mask self.logger.info("Single prediction completed") return {'M': M, 'D': D} except Exception as e: self.logger.error(f"An error occurred during prediction: {str(e)}") raise def predict_with_uncertainty(self, img1_path: str, img2_path: str, img3_path: str, n_samples=50) -> dict: """ Predict with Monte Carlo sampling, including uncertainty estimation. """ self.logger.info(f"Starting Monte Carlo prediction (samples: {n_samples})...") try: # Process the input images. inputs, orig_size = self._process_images(img1_path, img2_path, img3_path) mask, _ = self.create_mask(img1_path, img2_path, img3_path) mask = torch.from_numpy(mask).unsqueeze(0) # [1, H, W] # Store Monte Carlo samples. M_samples = [] D_samples = [] var_M_samples = [] var_D_samples = [] # Monte Carlo sampling uses training mode to enable dropout. self.model.train() inputs = inputs.unsqueeze(0).to(self.device) # [1, 3, H, W] with torch.no_grad(): for _ in range(n_samples): outputs = self.model(inputs) M_samples.append(outputs['M_mean'].cpu().numpy()) D_samples.append(outputs['D_mean'].cpu().numpy()) var_M_samples.append(outputs['M_var'].cpu().numpy()) var_D_samples.append(outputs['D_var'].cpu().numpy()) # Convert to NumPy arrays. M_samples = np.concatenate(M_samples, axis=0) D_samples = np.concatenate(D_samples, axis=0) var_M_samples = np.concatenate(var_M_samples, axis=0) var_D_samples = np.concatenate(var_D_samples, axis=0) # Calculate means and uncertainties. M_mean = np.mean(M_samples, axis=0).squeeze() D_mean = np.mean(D_samples, axis=0).squeeze() var_M_mean = np.mean(var_M_samples, axis=0).squeeze() var_D_mean = np.mean(var_D_samples, axis=0).squeeze() # Calculate model and data uncertainties. M_model_uncertainty = np.sqrt(np.mean((M_samples - M_mean) ** 2, axis=0)).squeeze() D_model_uncertainty = np.sqrt(np.mean((D_samples - D_mean) ** 2, axis=0)).squeeze() M_data_uncertainty = np.sqrt(np.mean(var_M_samples, axis=0)).squeeze() D_data_uncertainty = np.sqrt(np.mean(var_D_samples, axis=0)).squeeze() M_model_uncertainty = M_model_uncertainty D_model_uncertainty = D_model_uncertainty M_data_uncertainty = M_data_uncertainty D_data_uncertainty = D_data_uncertainty M_total_uncertainty = (M_model_uncertainty ** 2 + M_data_uncertainty ** 2) D_total_uncertainty = (D_model_uncertainty ** 2 + D_data_uncertainty ** 2) # Apply the mask. mask = mask.squeeze().numpy() M_mean *= mask D_mean *= mask M_model_uncertainty *= mask D_model_uncertainty *= mask M_data_uncertainty *= mask D_data_uncertainty *= mask M_total_uncertainty *= mask D_total_uncertainty *= mask var_M_mean *= mask var_D_mean *= mask # Create a mask with the original dimensions. full_mask = np.ones(orig_size, dtype=np.float32) full_mask[(np.array(Image.open(img1_path)) == 0) & (np.array(Image.open(img2_path)) == 0) & (np.array(Image.open(img3_path)) == 0)] = 0 # Restore the original dimensions. M_mean = self._restore_to_original_size(M_mean, orig_size) D_mean = self._restore_to_original_size(D_mean, orig_size) M_model_uncertainty = self._restore_to_original_size(M_model_uncertainty, orig_size) D_model_uncertainty = self._restore_to_original_size(D_model_uncertainty, orig_size) M_data_uncertainty = self._restore_to_original_size(M_data_uncertainty, orig_size) D_data_uncertainty = self._restore_to_original_size(D_data_uncertainty, orig_size) M_total_uncertainty = self._restore_to_original_size(M_total_uncertainty, orig_size) D_total_uncertainty = self._restore_to_original_size(D_total_uncertainty, orig_size) var_M_mean = self._restore_to_original_size(var_M_mean, orig_size) var_D_mean = self._restore_to_original_size(var_D_mean, orig_size) # Apply the full mask to the restored results. M_mean *= full_mask D_mean *= full_mask M_model_uncertainty *= full_mask D_model_uncertainty *= full_mask M_data_uncertainty *= full_mask D_data_uncertainty *= full_mask M_total_uncertainty *= full_mask D_total_uncertainty *= full_mask var_M_mean *= full_mask var_D_mean *= full_mask # Visualize uncertainty. self._visualize_uncertainties({ 'M': M_mean, 'D': D_mean, 'M_model_uncertainty': M_model_uncertainty, 'D_model_uncertainty': D_model_uncertainty, 'M_data_uncertainty': M_data_uncertainty, 'D_data_uncertainty': D_data_uncertainty, 'M_total_uncertainty': M_total_uncertainty, 'D_total_uncertainty': D_total_uncertainty }) # Do not process the 3D sample arrays further. self.logger.info("Monte Carlo prediction completed") return { 'M': M_mean, 'D': D_mean, 'M_var': var_M_mean, 'D_var': var_D_mean, 'M_model_uncertainty': M_model_uncertainty, 'D_model_uncertainty': D_model_uncertainty, 'M_data_uncertainty': M_data_uncertainty, 'D_data_uncertainty': D_data_uncertainty, 'M_total_uncertainty':M_total_uncertainty, 'D_total_uncertainty':D_total_uncertainty } except Exception as e: self.logger.error(f"An error occurred during prediction: {str(e)}") raise finally: # Return to evaluation mode after sampling. self.model.eval() def _visualize_uncertainties(self, results: dict, save_dir: str = 'uncertainty_analysis'): """ Visualize the uncertainty-analysis results. Args: results: Value returned by predict_with_uncertainty. save_dir: Output directory. """ os.makedirs(save_dir, exist_ok=True) plt.figure(figsize=(20, 10)) # M prediction and uncertainty. plt.subplot(2, 4, 1) plt.imshow(results['M']) plt.colorbar() plt.title('Predicted M') plt.subplot(2, 4, 2) plt.imshow(results['M_model_uncertainty']) plt.colorbar() plt.title('Model Uncertainty (M)') plt.subplot(2, 4, 3) plt.imshow(results['M_data_uncertainty']) plt.colorbar() plt.title('Data Uncertainty (M)') plt.subplot(2, 4, 4) total_M = results['M_total_uncertainty'] plt.imshow(total_M) plt.colorbar() plt.title('Total Uncertainty (M)') # D prediction and uncertainty. plt.subplot(2, 4, 5) plt.imshow(results['D']) plt.colorbar() plt.title('Predicted D') plt.subplot(2, 4, 6) plt.imshow(results['D_model_uncertainty']) plt.colorbar() plt.title('Model Uncertainty (D)') plt.subplot(2, 4, 7) plt.imshow(results['D_data_uncertainty']) plt.colorbar() plt.title('Data Uncertainty (D)') plt.subplot(2, 4, 8) total_D = results['D_total_uncertainty'] plt.imshow(total_D) plt.colorbar() plt.title('Total Uncertainty (D)') plt.tight_layout() plt.savefig(os.path.join(save_dir, 'uncertainty_analysis.png')) plt.close() self.logger.info(f"Uncertainty analysis plot saved to: {save_dir}") def save_prediction(self, results, save_path: str, mode='single'): """ Save prediction results to a .mat file. Args: results: Prediction results as a tuple or dictionary. save_path: Output path. mode: Prediction mode, either 'single' or 'mc'. """ try: os.makedirs(os.path.dirname(save_path), exist_ok=True) if mode == 'single': io.savemat(save_path, results) else: # mode == 'mc' io.savemat(save_path, results) self.logger.info(f"Prediction results saved to: {save_path}") except Exception as e: self.logger.error(f"Failed to save results: {str(e)}") raise if __name__ == "__main__": # Create the predictor and set the checkpoint used for inference. predictor = Predictor( model_path="path/to/your/model_weights.pth", crop_size=(1360,672), device='cuda' ) # Set paths to the three input frames. img1_path = 'path/to/your/input/image1.png' img2_path = 'path/to/your/input/image2.png' img3_path = 'path/to/your/input/image3.png' # Single-prediction example. try: result = predictor.predict_single(img1_path, img2_path, img3_path) predictor.save_prediction(result, 'pre_uncertainty/single_result.mat', mode='single') except Exception as e: print(f"Single prediction failed: {str(e)}") # Display the mask. mask, orig_size = predictor.create_mask(img1_path, img2_path, img3_path) # Restore the mask to its original dimensions when cropping is enabled. if predictor.crop_height is not None and predictor.crop_width is not None: full_mask = np.ones(orig_size, dtype=np.float32) full_mask[(np.array(Image.open(img1_path)) == 0) & (np.array(Image.open(img2_path)) == 0) & (np.array(Image.open(img3_path)) == 0)] = 0 mask_to_save = full_mask else: mask_to_save = mask # Save the mask as an image. cv2.imwrite('pre_uncertainty/mask.png', mask_to_save*255) # Save the mask as a .mat file. io.savemat('pre_uncertainty/mask.mat', {'mask':mask_to_save}) plt.imshow(mask_to_save) # Monte Carlo sampling example. try: results = predictor.predict_with_uncertainty(img1_path, img2_path, img3_path, n_samples=50) print("\nMonte Carlo prediction result shapes:") for key, value in results.items(): print(f"{key}: {value.shape}") predictor.save_prediction(results, 'pre_uncertainty/mc_result.mat', mode='mc') predictor._visualize_uncertainties(results) except Exception as e: print(f"Monte Carlo prediction failed: {str(e)}")