improve preprocessing stability

dynamix/preprocessing_utilities.py

@@ -388,12 +388,12 @@ class BoxCoxTransformer:
 
 class Detrending:
     """
-    Applies exponential detrending to time series data.
+    Applies detrending model to time series data.
     """
     @staticmethod
-    def exp_model(t, params):
+    def trend_model(t, params):
         """
-        Exponential model for detrending
+        Model for detrending
         
         Args:
             t: Time points
@@ -408,7 +408,7 @@ class Detrending:
     @staticmethod
     def fit_objective(params, data):
         """
-        Objective function for exponential model fitting
+        Objective function for trend model fitting
         
         Args:
             params: Model parameters
@@ -418,13 +418,13 @@ class Detrending:
             Sum of squared errors
         """
         t = np.arange(1, len(data) + 1)
-        predicted = Detrending.exp_model(t, params)
+        predicted = Detrending.trend_model(t, params)
         return np.sum((data - predicted) ** 2)
     
     @staticmethod
     def apply_detrending(data):
         """
-        Apply exponential detrending to data
+        Apply trend model to data
         
         Args:
             data: Input data tensor of shape (seq_length, N)
@@ -434,6 +434,11 @@ class Detrending:
         """
         # Convert to numpy
         data_np, is_torch, device, dtype = TimeSeriesProcessor.to_numpy(data)
+
+        # Apply min max scaling for a more stable trend fit
+        _min = np.min(data_np)
+        _max = np.max(data_np)
+        data_scaled = (data_np - _min) / (_max - _min)
         
         seq_length, n_dims = data_np.shape
         detrended_data = np.zeros_like(data_np)
@@ -441,13 +446,13 @@ class Detrending:
         
         for dim in range(n_dims):
             # Define the objective function for this dimension
-            objective = lambda params: Detrending.fit_objective(params, data_np[:, dim])
+            objective = lambda params: Detrending.fit_objective(params, data_scaled[:, dim])
             
             # Initial parameter guess
-            initial_params = [0.0, 1.0, data_np[0,dim]]
+            initial_params = [0.0, 1.0, data_scaled[0,dim]]
             
             # Bounds for parameters
-            bounds = [(None, None), (0.1, 3.0), (None, None)]
+            bounds = [(None, None), (0.0, 3.0), (None, None)]
             
             # Optimize
             result = optimize.minimize(
@@ -456,17 +461,20 @@ class Detrending:
                 method='L-BFGS-B', 
                 bounds=bounds,
                 options={
-                    'maxiter': 1000,
+                    'maxiter': 5000,
                     'gtol': 1e-6,
                     'maxfun': 1500,
                     'maxcor': 10
                 }
             )
             optimal_params = np.round(result.x, 3)
+            #Adjust params to min max scale
+            optimal_params[0] = (_max - _min) * optimal_params[0]
+            optimal_params[2] = (_max - _min) * optimal_params[2] + _min
             
             # Calculate trend and detrend the data
             t = np.arange(1, seq_length + 1)
-            trend = Detrending.exp_model(t, optimal_params)
+            trend = Detrending.trend_model(t, optimal_params)
             detrended_data[:, dim] = data_np[:, dim] - trend
             
             # Store parameters for inverse transformation
@@ -502,7 +510,7 @@ class Detrending:
         # Add trend back to each dimension
         for dim in range(min(n_dims, len(detrending_params))):
             params = detrending_params[dim]
-            trend = Detrending.exp_model(t, params)
+            trend = Detrending.trend_model(t, params)
             data_np[:, dim] = data_np[:, dim] + trend
         
         # Convert back to torch if needed