Upload feature_engineering-py.py

feature_engineering-py.py

@@ -0,0 +1,259 @@
+"""
+Feature engineering functions to create additional features from input bands.
+These functions generate vegetation indices, texture features, and other derived features
+that the model expects beyond the 59 base bands.
+
+Author: najahpokkiri
+Date: 2025-05-17
+"""
+import numpy as np
+from sklearn.preprocessing import StandardScaler
+from sklearn.decomposition import PCA
+
+# Safe division function for indices
+def safe_divide(a, b, fill_value=0.0):
+    """Safe division that handles zeros in the denominator"""
+    a = np.asarray(a, dtype=np.float32)
+    b = np.asarray(b, dtype=np.float32)
+    
+    # Handle NaN/Inf in inputs
+    a = np.nan_to_num(a, nan=0.0, posinf=0.0, neginf=0.0)
+    b = np.nan_to_num(b, nan=1e-10, posinf=1e10, neginf=-1e10)
+    
+    mask = np.abs(b) < 1e-10
+    result = np.full_like(a, fill_value, dtype=np.float32)
+    if np.any(~mask):
+        result[~mask] = a[~mask] / b[~mask]
+    
+    return np.nan_to_num(result, nan=fill_value, posinf=fill_value, neginf=fill_value)
+
+def calculate_spectral_indices(image_data):
+    """Calculate common spectral indices from satellite bands"""
+    indices = {}
+    
+    # Assuming standard band order: B2(blue), B3(green), B4(red), B8(nir), etc.
+    # Adjust band indices based on your data
+    try:
+        # Extract key bands (adjust indices as needed for your data)
+        blue = image_data[1] if image_data.shape[0] > 1 else None
+        green = image_data[2] if image_data.shape[0] > 2 else None
+        red = image_data[3] if image_data.shape[0] > 3 else None
+        nir = image_data[7] if image_data.shape[0] > 7 else None
+        swir1 = image_data[9] if image_data.shape[0] > 9 else None
+        swir2 = image_data[10] if image_data.shape[0] > 10 else None
+        
+        # Calculate indices if bands are available
+        if red is not None and nir is not None:
+            # NDVI (Normalized Difference Vegetation Index)
+            indices['NDVI'] = safe_divide(nir - red, nir + red)
+            
+            if blue is not None and green is not None:
+                # EVI (Enhanced Vegetation Index)
+                indices['EVI'] = 2.5 * safe_divide(nir - red, nir + 6*red - 7.5*blue + 1)
+                
+                # SAVI (Soil Adjusted Vegetation Index)
+                indices['SAVI'] = 1.5 * safe_divide(nir - red, nir + red + 0.5)
+                
+                # NDWI (Normalized Difference Water Index)
+                indices['NDWI'] = safe_divide(green - nir, green + nir)
+        
+        if swir1 is not None and nir is not None:
+            # NDMI (Normalized Difference Moisture Index)
+            indices['NDMI'] = safe_divide(nir - swir1, nir + swir1)
+        
+        if swir2 is not None and nir is not None:
+            # NBR (Normalized Burn Ratio)
+            indices['NBR'] = safe_divide(nir - swir2, nir + swir2)
+            
+    except Exception as e:
+        print(f"Error calculating spectral indices: {e}")
+    
+    return indices
+
+def extract_texture_features(image_data):
+    """Extract texture features from key bands"""
+    texture_features = {}
+    
+    try:
+        from skimage.filters import sobel
+        
+        # Use NIR band for texture if available
+        key_band_idx = 7  # NIR band
+        if image_data.shape[0] > key_band_idx:
+            band = image_data[key_band_idx].copy()
+            
+            # Normalize for texture analysis
+            band_min, band_max = np.nanpercentile(band[~np.isnan(band)], [1, 99])
+            band_norm = np.clip((band - band_min) / (band_max - band_min + 1e-8), 0, 1)
+            
+            # Handle NaN values
+            band_norm = np.nan_to_num(band_norm)
+            
+            # Edge detection using Sobel filter
+            sobel_response = sobel(band_norm)
+            texture_features['Sobel'] = sobel_response
+            
+            # Try to use more advanced texture features if available
+            try:
+                from skimage.feature import graycomatrix, graycoprops
+                
+                # Convert to uint8 for GLCM
+                band_uint8 = (band_norm * 255).astype(np.uint8)
+                
+                # Use a small central patch for efficiency
+                size = min(32, band_uint8.shape[0], band_uint8.shape[1])
+                center_y, center_x = band_uint8.shape[0] // 2, band_uint8.shape[1] // 2
+                half_size = size // 2
+                patch = band_uint8[
+                    max(0, center_y - half_size):min(band_uint8.shape[0], center_y + half_size),
+                    max(0, center_x - half_size):min(band_uint8.shape[1], center_x + half_size)
+                ]
+                
+                if patch.size > 0:
+                    # Calculate GLCM
+                    glcm = graycomatrix(patch, [1], [0], levels=256, symmetric=True, normed=True)
+                    
+                    # Extract properties
+                    for prop in ['contrast', 'dissimilarity', 'homogeneity', 'energy']:
+                        value = float(graycoprops(glcm, prop)[0, 0])
+                        # Fill with scalar value
+                        texture_features[f'GLCM_{prop}'] = np.full_like(band, value)
+            
+            except Exception as e:
+                print(f"Advanced texture features not available: {e}")
+    
+    except Exception as e:
+        print(f"Error extracting texture features: {e}")
+    
+    return texture_features
+
+def calculate_spatial_features(image_data, indices):
+    """Calculate spatial context features like gradients"""
+    spatial_features = {}
+    
+    try:
+        # Use NIR band for gradient if available
+        key_band_idx = 7  # NIR band
+        if image_data.shape[0] > key_band_idx:
+            band = image_data[key_band_idx].copy()
+            band = np.nan_to_num(band)
+            
+            # Calculate gradients
+            grad_y, grad_x = np.gradient(band)
+            grad_magnitude = np.sqrt(grad_x**2 + grad_y**2)
+            spatial_features['Gradient'] = grad_magnitude
+        
+        # Calculate NDVI gradient if available
+        if 'NDVI' in indices:
+            ndvi = indices['NDVI']
+            ndvi = np.nan_to_num(ndvi)
+            
+            grad_y, grad_x = np.gradient(ndvi)
+            grad_magnitude = np.sqrt(grad_x**2 + grad_y**2)
+            spatial_features['NDVI_gradient'] = grad_magnitude
+    
+    except Exception as e:
+        print(f"Error calculating spatial features: {e}")
+    
+    return spatial_features
+
+def calculate_pca_features(image_data, n_components=10):
+    """Calculate PCA features from satellite bands"""
+    pca_features = {}
+    
+    try:
+        # Reshape to (pixels, bands)
+        orig_shape = image_data.shape
+        bands_reshaped = image_data.reshape(orig_shape[0], -1).T
+        
+        # Handle NaN values
+        valid_mask = ~np.any(np.isnan(bands_reshaped), axis=1)
+        bands_clean = bands_reshaped[valid_mask]
+        
+        if len(bands_clean) > 0:
+            # Apply PCA
+            scaler = StandardScaler()
+            bands_scaled = scaler.fit_transform(bands_clean)
+            
+            # Limit components to save memory
+            n_components = min(n_components, bands_scaled.shape[1], 25)
+            pca = PCA(n_components=n_components)
+            pca_result = pca.fit_transform(bands_scaled)
+            
+            # Map back to original shape
+            pca_full = np.zeros((bands_reshaped.shape[0], n_components))
+            pca_full[valid_mask] = pca_result
+            
+            # Reshape to original spatial dimensions
+            pca_reshaped = pca_full.reshape(orig_shape[1], orig_shape[2], n_components)
+            
+            # Store each component
+            for i in range(n_components):
+                pca_features[f'PCA_{i+1}'] = pca_reshaped[:, :, i]
+    
+    except Exception as e:
+        print(f"Error calculating PCA features: {e}")
+    
+    return pca_features
+
+def extract_all_features(image_data):
+    """
+    Extract all necessary features from input bands to match the 99 features
+    expected by the model.
+    
+    Args:
+        image_data: NumPy array of shape (bands, height, width)
+        
+    Returns:
+        features_array: NumPy array of shape (pixels, features)
+        valid_mask: Boolean mask indicating valid pixels
+    """
+    height, width = image_data.shape[1], image_data.shape[2]
+    
+    # Create valid pixel mask (no NaN or Inf values)
+    valid_mask = np.all(np.isfinite(image_data), axis=0)
+    
+    # Extract valid pixel coordinates
+    valid_y, valid_x = np.where(valid_mask)
+    n_valid_pixels = len(valid_y)
+    
+    # Calculate all feature types
+    indices = calculate_spectral_indices(image_data)
+    texture_features = extract_texture_features(image_data)
+    spatial_features = calculate_spatial_features(image_data, indices)
+    pca_features = calculate_pca_features(image_data, n_components=25)
+    
+    # Combine features
+    all_features = {}
+    
+    # 1. Original bands
+    for i in range(image_data.shape[0]):
+        all_features[f'Band_{i+1}'] = image_data[i]
+    
+    # 2. Add other feature types
+    all_features.update(indices)
+    all_features.update(texture_features)
+    all_features.update(spatial_features)
+    all_features.update(pca_features)
+    
+    # Get list of all feature names
+    feature_names = list(all_features.keys())
+    print(f"Generated {len(feature_names)} features: {feature_names}")
+    
+    # Create feature matrix for valid pixels
+    feature_matrix = np.zeros((n_valid_pixels, len(feature_names)), dtype=np.float32)
+    
+    for i, feature_name in enumerate(feature_names):
+        feature_data = all_features[feature_name]
+        # Extract values for valid pixels
+        if feature_data.ndim == 2:
+            feature_values = feature_data[valid_y, valid_x]
+        else:
+            # Handle case where feature is a constant or has different dimensions
+            feature_values = np.full(n_valid_pixels, feature_data)
+        
+        # Handle NaN values
+        feature_values = np.nan_to_num(feature_values, nan=0.0)
+        feature_matrix[:, i] = feature_values
+    
+    return feature_matrix, valid_mask, feature_names