Update feature_engineering.py

feature_engineering.py

@@ -14,6 +14,14 @@ import warnings
 warnings.filterwarnings('ignore')
 
 
+def safe_zscore(series, window=252, min_obs=30):
+    """Rolling z-score with fallback to 0 for unstable windows"""
+    mean = series.rolling(window, min_periods=min_obs).mean()
+    std = series.rolling(window, min_periods=min_obs).std()
+    z = (series - mean) / std
+    return z.fillna(0).clip(-3, 3)
+
+
 class IntegratedTheoryFeatures:
     """
     Transforms raw market data into theory-driven features combining:
@@ -24,6 +32,12 @@ class IntegratedTheoryFeatures:
     """
     
     def __init__(self, df):
+        # Validate critical columns
+        required = {'SP500', 'DGS10', 'Gold', 'VIX', 'UNRATE', 'CPIAUCSL'}
+        missing = required - set(df.columns)
+        if missing:
+            raise ValueError(f"Critical data missing: {missing}")
+        
         self.df = df.copy()
         self.features = pd.DataFrame(index=df.index)
         
@@ -37,9 +51,11 @@ class IntegratedTheoryFeatures:
                 self.df[f'{col}_ret{window}'] = self.df[col].pct_change(window)
                 # Volatility
                 self.df[f'{col}_vol{window}'] = self.df[col].pct_change().rolling(window).std()
-                # Momentum (rate of change acceleration)
-                self.df[f'{col}_mom{window}'] = self.df[col].pct_change(window) - self.df[col].pct_change(window).shift(window)
-        
+                # Momentum
+                self.df[f'{col}_mom{window}'] = (
+                    self.df[col].pct_change(window) - 
+                    self.df[col].pct_change(window).shift(window)
+                )
         return self
     
     def dalio_forces(self):
@@ -57,7 +73,7 @@ class IntegratedTheoryFeatures:
             hy_spread * 0.3
         )
         
-        # Force 2: Internal Conflict (inequality-driven)
+        # Force 2: Internal Conflict
         consumer_weakness = (self.df.get('Consumer_Discretionary', 0) / 
                             self.df.get('Consumer_Staples', 1)).pct_change(63) * -1
         unemployment_stress = self.df.get('UNRATE', pd.Series(0)).diff() * 2
@@ -102,7 +118,7 @@ class IntegratedTheoryFeatures:
             ai_momentum * 0.3
         )
         
-        # Master Composite (normalized)
+        # Master Composite
         dalio_components = [
             self.features['dalio_debt_cycle'] * 0.35,
             self.features['dalio_internal_conflict'] * 0.25,
@@ -113,18 +129,15 @@ class IntegratedTheoryFeatures:
         
         self.features['dalio_composite'] = pd.concat(dalio_components, axis=1).sum(axis=1)
         self.features['dalio_composite_norm'] = self._normalize(self.features['dalio_composite'])
-        
         return self
     
     def stevenson_inequality(self):
         """Gary Stevenson's Inequality Amplification Metrics"""
         print("Building Stevenson's inequality indicators...")
         
-        # Wealth Flow (money flowing to asset owners vs middle class)
         asset_rich = (self.df.get('Gold', 0) + 
                      self.df.get('Real_Estate', 0) + 
                      self.df.get('Growth_Stocks', 0)) / 3
-        
         middle_class = (self.df.get('Consumer_Staples', 0) + 
                        self.df.get('Regional_Banks', 0) + 
                        self.df.get('Small_Cap_Value', 0)) / 3
@@ -133,25 +146,20 @@ class IntegratedTheoryFeatures:
             asset_rich.pct_change(63) - middle_class.pct_change(63)
         )
         
-        # Consumption Gap (luxury vs mass market)
         luxury = self.df.get('Retail_Luxury', pd.Series(0)).pct_change(21)
         mass = (self.df.get('Restaurants', 0) + self.df.get('Retail', 0)) / 2
         mass = mass.pct_change(21)
-        
         self.features['inequality_consumption_gap'] = luxury - mass
         
-        # Credit Access Gap
         quality_credit = (self.df.get('Investment_Grade_Spread', 0) + 
                          self.df.get('Preferred_Stock', 0)) / 2
         junk_credit = (self.df.get('HYG', 0) + 
                       self.df.get('JNK', 0) + 
                       self.df.get('Emerging_Market_Debt', 0)) / 3
-        
         self.features['inequality_credit_access'] = (
             quality_credit.pct_change(63) - junk_credit.pct_change(63)
         )
         
-        # Master Inequality Score
         self.features['stevenson_inequality'] = (
             self.features['inequality_wealth_flow'] * 0.4 +
             self.features['inequality_consumption_gap'] * 0.3 +
@@ -159,11 +167,8 @@ class IntegratedTheoryFeatures:
         )
         self.features['stevenson_inequality_norm'] = self._normalize(self.features['stevenson_inequality'])
         
-        # Inequality Transmission (how stimulus flows to rich)
-        # High when asset prices rise faster than wages
         asset_inflation = (self.df.get('Gold', 0) + self.df.get('Real_Estate', 0)).pct_change(21)
-        wage_proxy = self.df.get('Staffing', pd.Series(0)).pct_change(21)  # Labor market proxy
-        
+        wage_proxy = self.df.get('Staffing', pd.Series(0)).pct_change(21)
         self.features['inequality_transmission'] = asset_inflation - wage_proxy
         
         return self
@@ -172,32 +177,24 @@ class IntegratedTheoryFeatures:
         """Peter Thiel's Monopoly vs Competition Indicators"""
         print("Building Thiel's monopoly indicators...")
         
-        # Cash Moat (tech vs credit-dependent sectors)
         tech_strength = self.df.get('Technology', 0)
         finance_strength = self.df.get('Financials', 1)
-        
         self.features['monopoly_cash_moat'] = (
             tech_strength.pct_change(63) - finance_strength.pct_change(63)
         )
         
-        # Network Effects (winner-take-all platforms)
         network_sectors = (self.df.get('Cloud_Computing', 0) * 0.4 +
                           self.df.get('Communication_Services', 0) * 0.3 +
                           self.df.get('Fintech', 0) * 0.3)
-        
         self.features['monopoly_network_effects'] = network_sectors.pct_change(63)
         
-        # Defensibility (stability = moat strength)
         tech_volatility = self.df.get('Technology', pd.Series(1)).pct_change().rolling(63).std()
         chip_strength = self.df.get('Semiconductors', pd.Series(0)).pct_change(63)
-        
-        # Inverse volatility (lower vol = stronger moat)
         self.features['monopoly_defensibility'] = (
-            (1 / (tech_volatility + 0.001)) * 0.01 +  # Normalize
+            (1 / (tech_volatility + 0.001)) * 0.01 +
             chip_strength * 0.5
         )
         
-        # Master Monopoly Score
         self.features['thiel_monopoly'] = (
             self.features['monopoly_cash_moat'] * 0.35 +
             self.features['monopoly_network_effects'] * 0.35 +
@@ -205,18 +202,14 @@ class IntegratedTheoryFeatures:
         )
         self.features['thiel_monopoly_norm'] = self._normalize(self.features['thiel_monopoly'])
         
-        # Monopoly Immunity Test (tech ignoring rate moves)
         tech_return = self.df.get('Technology', pd.Series(0)).pct_change(21)
-        rate_change = self.df.get('DGS10', pd.Series(0)).diff() * -1  # Inverse (cuts = positive)
-        
+        rate_change = self.df.get('DGS10', pd.Series(0)).diff() * -1
         self.features['monopoly_immunity'] = tech_return / (rate_change.abs() + 0.001)
         
-        # Tech Concentration (narrow leadership = bubble risk)
         specialized = (self.df.get('Semiconductors', 0) + 
                       self.df.get('Cloud_Computing', 0) + 
                       self.df.get('Robotics_AI', 0)) / 3
         broad_tech = self.df.get('Technology', 1)
-        
         self.features['tech_concentration'] = specialized / broad_tech
         
         return self
@@ -225,49 +218,38 @@ class IntegratedTheoryFeatures:
         """Jeffrey Gundlach's Debt Reckoning and Paradigm Shift Signals"""
         print("Building Gundlach's reckoning indicators...")
         
-        # Yield Anomaly (yields rising post-cuts = fiscal dominance)
         fed_proxy = self.df.get('DGS3MO', pd.Series(0))
         long_yield = self.df.get('DGS10', pd.Series(0))
-        
-        # Detect cuts (3mo falling) and measure 10Y response
         fed_cutting = fed_proxy.diff() < -0.05
         yield_rising = long_yield.diff() > 0
-        
         self.features['gundlach_yield_anomaly'] = (
             (fed_cutting & yield_rising).astype(float) +
-            (long_yield - fed_proxy)  # Curve steepening
+            (long_yield - fed_proxy)
         )
         
-        # Flight-to-Quality Shift (gold vs Treasuries)
         gold_return = self.df.get('Gold', pd.Series(0)).pct_change(21)
         treasury_return = self.df.get('US_Treasuries_Long', pd.Series(1)).pct_change(21)
-        
         self.features['gundlach_flight_shift'] = gold_return / (treasury_return + 0.001)
         
-        # Capital Reversal (dollar weakness + EM outperformance)
         dollar_weak = self.df.get('DXY', pd.Series(0)).pct_change(21) * -1
         em_outperform = (self.df.get('Emerging_Markets', 0) + self.df.get('Europe', 0)) / 2
         em_outperform = em_outperform.pct_change(21)
         sp_return = self.df.get('SP500', pd.Series(0)).pct_change(21)
-        
         self.features['gundlach_capital_reversal'] = (
             dollar_weak * 0.5 +
             (em_outperform - sp_return) * 0.5
         )
         
-        # Private Credit Risk (2007 CDO echo)
         regional_stress = (self.df.get('Regional_Banks', 0) / 
                           self.df.get('Financials', 1)).pct_change(21)
         mortgage_reit_stress = self.df.get('Mortgage_REITs', pd.Series(0)).pct_change(21)
         real_estate_vol = self.df.get('Real_Estate', pd.Series(1)).pct_change().rolling(21).std() * 100
-        
         self.features['gundlach_private_credit_risk'] = (
-            regional_stress * -0.4 +  # Decline = stress
+            regional_stress * -0.4 +
             mortgage_reit_stress * -0.3 +
             real_estate_vol * 0.3
         )
         
-        # Master Reckoning Score
         self.features['gundlach_reckoning'] = (
             self.features['gundlach_yield_anomaly'] * 0.30 +
             self.features['gundlach_flight_shift'] * 0.25 +
@@ -275,48 +257,40 @@ class IntegratedTheoryFeatures:
             self.features['gundlach_private_credit_risk'] * 0.20
         )
         self.features['gundlach_reckoning_norm'] = self._normalize(self.features['gundlach_reckoning'])
-        
         return self
     
     def geopolitical_indicators(self):
         """Regional conflict and energy transition signals"""
         print("Building geopolitical indicators...")
         
-        # Middle East Risk
         oil_volatility = self.df.get('Oil', pd.Series(1)).pct_change().rolling(3).std() * 100
         defense_spike = self.df.get('Defense_Stocks', pd.Series(0)).pct_change(5)
         gold_haven = self.df.get('Gold_Safe_Haven', pd.Series(0)).pct_change(5)
-        
         self.features['middle_east_risk'] = (
             oil_volatility * 0.4 +
             defense_spike * 0.3 +
             gold_haven * 0.3
         )
         
-        # Europe Risk
         gas_volatility = self.df.get('NaturalGas', pd.Series(1)).pct_change().rolling(5).std() * 100
         europe_decline = self.df.get('Europe', pd.Series(0)).pct_change(21) * -1
-        swiss_franc_strength = self.df.get('Swiss_Franc', pd.Series(0)).pct_change(21) * -1  # Inverse quote
-        
+        swiss_franc_strength = self.df.get('Swiss_Franc', pd.Series(0)).pct_change(21) * -1
         self.features['europe_risk'] = (
             gas_volatility * 0.5 +
             europe_decline * 0.3 +
             swiss_franc_strength * 0.2
         )
         
-        # Asia Risk
         chip_stress = self.df.get('Semiconductors', pd.Series(1)).pct_change().rolling(21).std() * 100
         taiwan_korea = (self.df.get('Taiwan', 0) + self.df.get('South_Korea', 0)) / 2
         china_diverge = taiwan_korea.pct_change(21) - self.df.get('China', pd.Series(0)).pct_change(21)
         rare_earth = self.df.get('Rare_Earth', pd.Series(0)).pct_change(21)
-        
         self.features['asia_risk'] = (
             chip_stress * 0.4 +
             china_diverge * 0.3 +
             rare_earth * 0.3
         )
         
-        # Overall Geopolitical Risk
         self.features['geopolitical_risk'] = (
             self.features['middle_east_risk'] * 0.4 +
             self.features['europe_risk'] * 0.3 +
@@ -324,54 +298,44 @@ class IntegratedTheoryFeatures:
         )
         self.features['geopolitical_risk_norm'] = self._normalize(self.features['geopolitical_risk'])
         
-        # Energy Transition Indicators
         uranium_momentum = self.df.get('Uranium', pd.Series(0)).pct_change(63)
         clean_momentum = self.df.get('Clean_Energy', pd.Series(0)).pct_change(63)
         oil_decline = self.df.get('Oil', pd.Series(0)).pct_change(252) * -1
-        
         self.features['energy_transition'] = (
             uranium_momentum * 0.5 +
             clean_momentum * 0.3 +
             oil_decline * 0.2
         )
-        
         return self
     
     def cross_asset_features(self):
         """Advanced cross-asset relationships"""
         print("Building cross-asset features...")
         
-        # Flight-to-Quality Ratio
         defensive = (self.df.get('Gold', 0) + 
                     self.df.get('Utilities', 0) + 
                     self.df.get('Healthcare', 0)) / 3
         risk_on = (self.df.get('Technology', 0) + 
                   self.df.get('Consumer_Discretionary', 0) + 
                   self.df.get('Real_Estate', 0)) / 3
-        
         self.features['flight_ratio'] = defensive / (risk_on + 0.001)
         
-        # Credit Contagion Spread
         regional_vs_broad = (self.df.get('Regional_Banks', 0) - 
                             self.df.get('Financials', 0))
         mortgage_vs_reit = (self.df.get('Mortgage_REITs', 0) - 
                            self.df.get('REITs', 0))
         em_vs_ig = (self.df.get('Emerging_Market_Debt', 0) - 
                    self.df.get('Investment_Grade_Spread', 0))
-        
         self.features['credit_contagion'] = (
             regional_vs_broad.pct_change(21) +
             mortgage_vs_reit.pct_change(21) +
             em_vs_ig.pct_change(21)
         ) / 3
         
-        # VIX Amplification
         vix = self.df.get('VIX', pd.Series(20))
         vix_historical_avg = vix.rolling(252).mean()
         geo_max = self.features[['middle_east_risk', 'europe_risk', 'asia_risk']].max(axis=1)
-        
         self.features['geo_amplification'] = geo_max * (vix / vix_historical_avg)
-        
         return self
     
     def scenario_probabilities(self):
@@ -381,19 +345,18 @@ class IntegratedTheoryFeatures:
         # Scenario 1: Credit Collapse
         self.features['prob_credit_collapse'] = (
             self.features['gundlach_reckoning_norm'] * 0.4 +
-            self.features['gundlach_private_credit_risk'] / self.features['gundlach_private_credit_risk'].std() * 0.1 * 0.3 +
-            self.features['dalio_debt_cycle'] / self.features['dalio_debt_cycle'].std() * 0.1 * 0.3
+            safe_zscore(self.features['gundlach_private_credit_risk']) * 0.03 +
+            safe_zscore(self.features['dalio_debt_cycle']) * 0.03
         )
         self.features['prob_credit_collapse'] = np.clip(self.features['prob_credit_collapse'], 0, 1)
         
         # Scenario 2: Stagflation
         inflation_high = (self.df.get('CPIAUCSL', pd.Series(0)).pct_change(12) * 100 > 2.5).astype(float)
         unemployment_rising = (self.df.get('UNRATE', pd.Series(0)).diff() > 0).astype(float)
-        
         self.features['prob_stagflation'] = (
             (inflation_high * unemployment_rising) * 0.3 +
-            self.features['dalio_external_conflict'] / self.features['dalio_external_conflict'].std() * 0.1 * 0.3 +
-            self.features['gundlach_capital_reversal'] / self.features['gundlach_capital_reversal'].std() * 0.1 * 0.2 +
+            safe_zscore(self.features['dalio_external_conflict']) * 0.03 +
+            safe_zscore(self.features['gundlach_capital_reversal']) * 0.02 +
             self.features['stevenson_inequality_norm'] * 0.2
         )
         self.features['prob_stagflation'] = np.clip(self.features['prob_stagflation'], 0, 1)
@@ -401,17 +364,14 @@ class IntegratedTheoryFeatures:
         # Scenario 3: Tech Monopoly Boom
         self.features['prob_tech_boom'] = (
             self.features['thiel_monopoly_norm'] * 0.4 +
-            (self.features['dalio_tech_force'] - self.features['dalio_debt_cycle']) / 
-            (self.features['dalio_tech_force'].std() + 0.001) * 0.1 * 0.3 +
-            self.features['energy_transition'] / (self.features['energy_transition'].std() + 0.001) * 0.1 * 0.2 +
+            safe_zscore(self.features['dalio_tech_force'] - self.features['dalio_debt_cycle']) * 0.03 +
+            safe_zscore(self.features['energy_transition']) * 0.02 +
             (self.df.get('China_Tech', pd.Series(0)).pct_change(63) < 
              self.df.get('Technology', pd.Series(0)).pct_change(63)).astype(float) * 0.1
         )
         self.features['prob_tech_boom'] = np.clip(self.features['prob_tech_boom'], 0, 1)
         
-        # Scenario 4: Controlled Reset (low probability without policy action)
-        self.features['prob_controlled_reset'] = 0.05  # Baseline, would need policy signals
-        
+        self.features['prob_controlled_reset'] = 0.05
         return self
     
     def regime_detection(self):
@@ -419,37 +379,24 @@ class IntegratedTheoryFeatures:
         print("Detecting market regimes...")
         
         def classify_regime(row):
-            # Crisis conditions
-            if (row['gundlach_reckoning_norm'] > 0.6 and 
-                row['prob_credit_collapse'] > 0.5):
+            if (row['gundlach_reckoning_norm'] > 0.6 and row['prob_credit_collapse'] > 0.5):
                 return 'CRISIS'
-            
-            # Tech Monopoly Dominance
             elif row['thiel_monopoly_norm'] > 0.7:
                 return 'TECH_MONOPOLY'
-            
-            # Inequality Trap (stagflation)
-            elif (row['stevenson_inequality_norm'] > 0.6 and 
-                  row['prob_stagflation'] > 0.4):
+            elif (row['stevenson_inequality_norm'] > 0.6 and row['prob_stagflation'] > 0.4):
                 return 'INEQUALITY_TRAP'
-            
-            # Geopolitical Shock
             elif row['geopolitical_risk_norm'] > 0.7:
                 return 'GEOPOLITICAL_SHOCK'
-            
-            # Default: Transition phase
             else:
                 return 'TRANSITION'
         
         self.features['regime'] = self.features.apply(classify_regime, axis=1)
-        
         return self
     
     def dimensionality_reduction(self):
         """Apply PCA to reduce feature space"""
         print("Applying dimensionality reduction...")
         
-        # Define feature groups for PCA
         debt_cols = [c for c in self.features.columns if 'dalio_debt' in c or 'gundlach' in c]
         inequality_cols = [c for c in self.features.columns if 'inequality' in c or 'stevenson' in c]
         geo_cols = [c for c in self.features.columns if 'risk' in c or 'middle_east' in c or 'europe' in c or 'asia' in c]
@@ -458,47 +405,32 @@ class IntegratedTheoryFeatures:
         for name, cols in [('debt', debt_cols), ('inequality', inequality_cols), 
                           ('geo', geo_cols), ('tech', tech_cols)]:
             if len(cols) > 0:
-                # Get data and drop NaNs
                 data = self.features[cols].dropna()
-                
-                if len(data) > 10:  # Need sufficient data
-                    # Standardize
+                if len(data) > 10:
                     scaler = StandardScaler()
                     data_scaled = scaler.fit_transform(data)
-                    
-                    # PCA
                     pca = PCA(n_components=min(2, len(cols)))
                     pcs = pca.fit_transform(data_scaled)
-                    
-                    # Add back
                     for i in range(pcs.shape[1]):
                         self.features.loc[data.index, f'{name}_PC{i+1}'] = pcs[:, i]
-        
         return self
     
     def _calculate_dollar_anomaly(self):
-        """Detect dollar weakness during stock corrections (40-year anomaly)"""
         sp_correction = self.df.get('SP500', pd.Series(0)).pct_change(5) < -0.05
         dollar_weakness = self.df.get('DXY', pd.Series(0)).pct_change(5) < 0
-        
         return (sp_correction & dollar_weakness).astype(float)
     
     def _calculate_asia_tension(self):
-        """Taiwan-China divergence as tension proxy"""
         taiwan = self.df.get('Taiwan', pd.Series(0))
         china = self.df.get('China', pd.Series(0))
-        
         return (taiwan.pct_change(21) - china.pct_change(21)).fillna(0)
     
     def _normalize(self, series, window=252):
-        """Rolling z-score normalization"""
         rolling_mean = series.rolling(window, min_periods=20).mean()
         rolling_std = series.rolling(window, min_periods=20).std()
-        
-        return ((series - rolling_mean) / (rolling_std + 0.001)).clip(-3, 3) / 3  # Scale to -1, 1
+        return ((series - rolling_mean) / (rolling_std + 0.001)).clip(-3, 3) / 3
     
     def build_all_features(self):
-        """Run complete feature engineering pipeline"""
         print("\n" + "="*80)
         print("INTEGRATED THEORY FEATURE ENGINEERING")
         print("="*80 + "\n")
@@ -534,26 +466,24 @@ class IntegratedTheoryFeatures:
 
 
 def main():
-    """Main execution function"""
     import argparse
-    
     parser = argparse.ArgumentParser(description='Integrated Market Theory Feature Engineering')
     parser.add_argument('--input', default='unified_market_data.csv', 
                        help='Input CSV file from geo_macro.py')
     parser.add_argument('--output', default='enhanced_market_features.csv',
                        help='Output CSV file with engineered features')
-    
     args = parser.parse_args()
     
-    # Load data
     print(f"Loading data from {args.input}...")
     df = pd.read_csv(args.input, index_col=0, parse_dates=True)
     print(f"Loaded {len(df)} rows, {len(df.columns)} columns")
     print(f"Date range: {df.index.min()} to {df.index.max()}")
     
-    # Build features
     engine = IntegratedTheoryFeatures(df)
     features = engine.build_all_features()
     
-    # Save
-    features.to_csv(args.output)
+    features.to_csv(args.output)  # ✅ FIXED: added missing parenthesis
+
+
+if __name__ == "__main__":
+    main()