Update feature_engineering.py

feature_engineering.py

@@ -1,5 +1,5 @@
 """
-Integrated Market Theory - Feature Engineering Pipeline
+Integrated Market Theory - Enhanced Feature Engineering Pipeline
 Generates transparent, theory-driven features for regime detection and strategic allocation.
 
 Usage:
@@ -12,15 +12,23 @@ from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
 def safe_zscore(series, window=252, min_obs=30):
+    """Calculate rolling z-score with safety bounds"""
     mean = series.rolling(window, min_periods=min_obs).mean()
     std = series.rolling(window, min_periods=min_obs).std()
-    z = (series - mean) / std
+    z = (series - mean) / (std + 1e-8)
     return z.fillna(0).clip(-3, 3)
 
 def normalize(series, window=252):
+    """Normalize series to [-1, 1] range using rolling statistics"""
     rolling_mean = series.rolling(window, min_periods=20).mean()
     rolling_std = series.rolling(window, min_periods=20).std()
-    return ((series - rolling_mean) / (rolling_std + 0.001)).clip(-3, 3) / 3
+    normalized = (series - rolling_mean) / (rolling_std + 1e-8)
+    return normalized.fillna(0).clip(-3, 3) / 3
+
+def safe_divide(numerator, denominator, fill_value=0):
+    """Safe division with handling for zero/NaN denominator"""
+    result = numerator / (denominator + 1e-8)
+    return result.replace([np.inf, -np.inf], fill_value).fillna(fill_value)
 
 class IntegratedTheoryFeatures:
     def __init__(self, df):
@@ -32,40 +40,74 @@ class IntegratedTheoryFeatures:
         self.features = pd.DataFrame(index=df.index)
 
     def dalio_forces(self):
-        # Debt Cycle
+        """Ray Dalio's Five Forces Framework"""
+        # 1. Debt Cycle
         yield_curve = self.df.get('DGS10', 0) - self.df.get('DGS2', 0)
         inflation_mom = self.df.get('CPIAUCSL', pd.Series(0)).pct_change(12) * 100
         hy_spread = self.df.get('BAMLH0A0HYM2', pd.Series(0)) / 100
-        self.features['dalio_debt_cycle'] = yield_curve * 0.3 + inflation_mom * 0.4 + hy_spread * 0.3
+        self.features['dalio_debt_cycle'] = (
+            yield_curve * 0.3 + 
+            inflation_mom * 0.4 + 
+            hy_spread * 0.3
+        )
 
-        # Internal Conflict
-        consumer_weakness = (self.df.get('Consumer_Discretionary', 0) / self.df.get('Consumer_Staples', 1)).pct_change(63) * -1
+        # 2. Internal Conflict (Inequality & Social Stress)
+        consumer_weakness = safe_divide(
+            self.df.get('Consumer_Discretionary', 0),
+            self.df.get('Consumer_Staples', 1)
+        ).pct_change(63) * -1
+        
         unemployment_stress = self.df.get('UNRATE', pd.Series(0)).diff() * 2
-        small_large_gap = (self.df.get('Small_Cap_Value', 0) / self.df.get('SP500', 1)).pct_change(63) * -1
-        self.features['dalio_internal_conflict'] = consumer_weakness * 0.4 + unemployment_stress * 0.3 + small_large_gap * 0.3
+        
+        small_large_gap = safe_divide(
+            self.df.get('Small_Cap_Value', 0),
+            self.df.get('SP500', 1)
+        ).pct_change(63) * -1
+        
+        self.features['dalio_internal_conflict'] = (
+            consumer_weakness * 0.4 + 
+            unemployment_stress * 0.3 + 
+            small_large_gap * 0.3
+        )
 
-        # External Conflict
+        # 3. External Conflict (Geopolitical)
         defense_momentum = self.df.get('Defense_Stocks', pd.Series(0)).pct_change(21)
+        
         sp_corr = self.df.get('SP500', pd.Series(0)).pct_change(5) < -0.05
         dollar_weak = self.df.get('DXY', pd.Series(0)).pct_change(5) < 0
         dollar_anomaly = (sp_corr & dollar_weak).astype(float)
+        
         taiwan = self.df.get('Taiwan', pd.Series(0))
         china = self.df.get('China', pd.Series(0))
         china_taiwan_tension = (taiwan.pct_change(21) - china.pct_change(21)).fillna(0)
-        self.features['dalio_external_conflict'] = defense_momentum * 0.4 + dollar_anomaly * 0.3 + china_taiwan_tension * 0.3
+        
+        self.features['dalio_external_conflict'] = (
+            defense_momentum * 0.4 + 
+            dollar_anomaly * 0.3 + 
+            china_taiwan_tension * 0.3
+        )
 
-        # Nature
+        # 4. Nature Force (Climate & Resources)
         water_stress = self.df.get('Water', pd.Series(0)).pct_change(63)
         ag_vol = self.df.get('Agricultural', pd.Series(0)).pct_change().rolling(63).std() * 100
         self.features['dalio_nature_force'] = water_stress * 0.6 + ag_vol * 0.4
 
-        # Tech Force
-        tech_outperform = (self.df.get('Technology', 0) / self.df.get('SP500', 1)).pct_change(21)
+        # 5. Technology Force
+        tech_outperform = safe_divide(
+            self.df.get('Technology', 0),
+            self.df.get('SP500', 1)
+        ).pct_change(21)
+        
         cloud_mom = self.df.get('Cloud_Computing', pd.Series(0)).pct_change(63)
         ai_mom = self.df.get('Robotics_AI', pd.Series(0)).pct_change(63)
-        self.features['dalio_tech_force'] = tech_outperform * 0.4 + cloud_mom * 0.3 + ai_mom * 0.3
+        
+        self.features['dalio_tech_force'] = (
+            tech_outperform * 0.4 + 
+            cloud_mom * 0.3 + 
+            ai_mom * 0.3
+        )
 
-        # Composite
+        # Composite Score
         comp = (
             self.features['dalio_debt_cycle'] * 0.35 +
             self.features['dalio_internal_conflict'] * 0.25 +
@@ -77,16 +119,39 @@ class IntegratedTheoryFeatures:
         return self
 
     def stevenson_inequality(self):
-        asset_rich = (self.df.get('Gold', 0) + self.df.get('Real_Estate', 0) + self.df.get('Growth_Stocks', 0)) / 3
-        middle_class = (self.df.get('Consumer_Staples', 0) + self.df.get('Regional_Banks', 0) + self.df.get('Small_Cap_Value', 0)) / 3
+        """Betsey Stevenson's Economic Inequality Framework"""
+        # Wealth Concentration
+        asset_rich = (
+            self.df.get('Gold', 0) + 
+            self.df.get('Real_Estate', 0) + 
+            self.df.get('Growth_Stocks', 0)
+        ) / 3
+        
+        middle_class = (
+            self.df.get('Consumer_Staples', 0) + 
+            self.df.get('Regional_Banks', 0) + 
+            self.df.get('Small_Cap_Value', 0)
+        ) / 3
+        
         wealth_flow = asset_rich.pct_change(63) - middle_class.pct_change(63)
 
+        # Consumer Spending Gap
         luxury = self.df.get('Retail_Luxury', pd.Series(0)).pct_change(21)
-        mass = ((self.df.get('Restaurants', 0) + self.df.get('Retail', 0)) / 2).pct_change(21)
-        cons_gap = luxury - mass
-
-        quality = (self.df.get('Investment_Grade_Spread', 0) + self.df.get('Preferred_Stock', 0)) / 2
-        junk = (self.df.get('HYG', 0) + self.df.get('JNK', 0) + self.df.get('Emerging_Market_Debt', 0)) / 3
+        mass_market = (
+            (self.df.get('Restaurants', 0) + self.df.get('Retail', 0)) / 2
+        ).pct_change(21)
+        cons_gap = luxury - mass_market
+
+        # Credit Access Gap
+        quality = (
+            self.df.get('Investment_Grade_Spread', 0) + 
+            self.df.get('Preferred_Stock', 0)
+        ) / 2
+        junk = (
+            self.df.get('HYG', 0) + 
+            self.df.get('JNK', 0) + 
+            self.df.get('Emerging_Market_Debt', 0)
+        ) / 3
         credit_gap = quality.pct_change(63) - junk.pct_change(63)
 
         self.features['stevenson_inequality_norm'] = normalize(
@@ -95,19 +160,23 @@ class IntegratedTheoryFeatures:
         return self
 
     def thiel_monopoly(self):
+        """Peter Thiel's Zero to One / Monopoly Framework"""
+        # Cash Flow Moats
         tech = self.df.get('Technology', 0)
         finance = self.df.get('Financials', 1)
         cash_moat = tech.pct_change(63) - finance.pct_change(63)
 
+        # Network Effects
         network = (
             self.df.get('Cloud_Computing', 0) * 0.4 +
             self.df.get('Communication_Services', 0) * 0.3 +
             self.df.get('Fintech', 0) * 0.3
         ).pct_change(63)
 
+        # Defensibility (Low volatility + semiconductor dominance)
         tech_vol = self.df.get('Technology', pd.Series(1)).pct_change().rolling(63).std()
         chip = self.df.get('Semiconductors', pd.Series(0)).pct_change(63)
-        defensibility = (1 / (tech_vol + 0.001)) * 0.01 + chip * 0.5
+        defensibility = safe_divide(1, tech_vol) * 0.01 + chip * 0.5
 
         self.features['thiel_monopoly_norm'] = normalize(
             cash_moat * 0.35 + network * 0.35 + defensibility * 0.30
@@ -115,25 +184,45 @@ class IntegratedTheoryFeatures:
         return self
 
     def gundlach_reckoning(self):
+        """Jeffrey Gundlach's Debt Reckoning Framework"""
+        # Yield Anomalies
         fed = self.df.get('DGS3MO', pd.Series(0))
         teny = self.df.get('DGS10', pd.Series(0))
-        yield_anomaly = ((fed.diff() < -0.05) & (teny.diff() > 0)).astype(float) + (teny - fed)
+        yield_anomaly = (
+            ((fed.diff() < -0.05) & (teny.diff() > 0)).astype(float) + 
+            (teny - fed)
+        )
 
+        # Flight to Safety Shift (Gold vs Bonds)
         gold_ret = self.df.get('Gold', pd.Series(0)).pct_change(21)
         tlt_ret = self.df.get('US_Treasuries_Long', pd.Series(1)).pct_change(21)
-        flight_shift = gold_ret / (tlt_ret + 0.001)
+        flight_shift = safe_divide(gold_ret, tlt_ret)
 
+        # Capital Flow Reversal
         dxy_weak = self.df.get('DXY', pd.Series(0)).pct_change(21) * -1
         em = (self.df.get('Emerging_Markets', 0) + self.df.get('Europe', 0)) / 2
         em_out = em.pct_change(21)
         sp_ret = self.df.get('SP500', pd.Series(0)).pct_change(21)
         capital_reversal = dxy_weak * 0.5 + (em_out - sp_ret) * 0.5
-
-        reg_banks = (self.df.get('Regional_Banks', 0) / self.df.get('Financials', 1)).pct_change(21)
+        self.features['gundlach_capital_reversal'] = capital_reversal
+
+        # Private Credit Risk
+        reg_banks = safe_divide(
+            self.df.get('Regional_Banks', 0),
+            self.df.get('Financials', 1)
+        ).pct_change(21)
+        
         mortgage_reit = self.df.get('Mortgage_REITs', pd.Series(0)).pct_change(21)
         real_estate_vol = self.df.get('Real_Estate', pd.Series(1)).pct_change().rolling(21).std() * 100
-        private_credit_risk = reg_banks * -0.4 + mortgage_reit * -0.3 + real_estate_vol * 0.3
+        
+        private_credit_risk = (
+            reg_banks * -0.4 + 
+            mortgage_reit * -0.3 + 
+            real_estate_vol * 0.3
+        )
+        self.features['gundlach_private_credit_risk'] = private_credit_risk
 
+        # Composite
         reckoning = (
             yield_anomaly * 0.30 +
             flight_shift * 0.25 +
@@ -141,20 +230,23 @@ class IntegratedTheoryFeatures:
             private_credit_risk * 0.20
         )
         self.features['gundlach_reckoning_norm'] = normalize(reckoning)
-        self.features['gundlach_private_credit_risk'] = private_credit_risk
         return self
 
     def geopolitical_indicators(self):
+        """Enhanced Geopolitical Risk Indicators"""
+        # Middle East Risk
         oil_vol = self.df.get('Oil', pd.Series(1)).pct_change().rolling(3).std() * 100
         def_spike = self.df.get('Defense_Stocks', pd.Series(0)).pct_change(5)
         gold_haven = self.df.get('Gold_Safe_Haven', pd.Series(0)).pct_change(5)
         me_risk = oil_vol * 0.4 + def_spike * 0.3 + gold_haven * 0.3
 
+        # Europe Risk
         gas_vol = self.df.get('NaturalGas', pd.Series(1)).pct_change().rolling(5).std() * 100
         eu_decline = self.df.get('Europe', pd.Series(0)).pct_change(21) * -1
         chf_str = self.df.get('Swiss_Franc', pd.Series(0)).pct_change(21) * -1
         eu_risk = gas_vol * 0.5 + eu_decline * 0.3 + chf_str * 0.2
 
+        # Asia-Pacific Risk
         chip_stress = self.df.get('Semiconductors', pd.Series(1)).pct_change().rolling(21).std() * 100
         tw_kr = (self.df.get('Taiwan', 0) + self.df.get('South_Korea', 0)) / 2
         china_div = tw_kr.pct_change(21) - self.df.get('China', pd.Series(0)).pct_change(21)
@@ -167,10 +259,11 @@ class IntegratedTheoryFeatures:
         return self
 
     def scenario_probabilities(self):
+        """Calculate probabilities for key scenarios"""
         f = self.features
         df = self.df
 
-        # Credit Collapse
+        # Credit Collapse Probability
         f['prob_credit_collapse'] = np.clip(
             f['gundlach_reckoning_norm'] * 0.4 +
             safe_zscore(f['gundlach_private_credit_risk']) * 0.03 +
@@ -178,23 +271,25 @@ class IntegratedTheoryFeatures:
             0, 1
         )
 
-        # Stagflation
+        # Stagflation Probability
         inflation_high = (df['CPIAUCSL'].pct_change(12) * 100 > 2.5).astype(float)
         unemp_rising = (df['UNRATE'].diff() > 0).astype(float)
         f['prob_stagflation'] = np.clip(
             (inflation_high & unemp_rising) * 0.3 +
             safe_zscore(f['dalio_external_conflict']) * 0.03 +
-            safe_zscore(f['gundlach_capital_reversal']) * 0.02 +
+            safe_zscore(f.get('gundlach_capital_reversal', pd.Series(0))) * 0.02 +
             f['stevenson_inequality_norm'] * 0.2,
             0, 1
         )
 
-        # Tech Boom
-        china_tech_lag = (df.get('China_Tech', pd.Series(0)).pct_change(63) < df.get('Technology', pd.Series(0)).pct_change(63)).astype(float)
+        # Tech Boom Probability
+        china_tech = df.get('China_Tech', pd.Series(0)).pct_change(63)
+        tech = df.get('Technology', pd.Series(0)).pct_change(63)
+        china_tech_lag = (china_tech < tech).astype(float)
+        
         f['prob_tech_boom'] = np.clip(
             f['thiel_monopoly_norm'] * 0.4 +
             safe_zscore(f['dalio_tech_force'] - f['dalio_debt_cycle']) * 0.03 +
-            safe_zscore(f.get('energy_transition', pd.Series(0))) * 0.02 +
             china_tech_lag * 0.1,
             0, 1
         )
@@ -202,14 +297,25 @@ class IntegratedTheoryFeatures:
         return self
 
     def regime_flags(self):
+        """Determine market regime flags"""
         f = self.features
+        
         # Binary regime flags
-        f['debt_unsustainable'] = ((f['gundlach_reckoning_norm'] > 0.5) & (f['prob_credit_collapse'] > 0.3)).astype(int)
-        f['inequality_trap'] = ((f['stevenson_inequality_norm'] > 0.6) & (f['prob_stagflation'] > 0.4)).astype(int)
+        f['debt_unsustainable'] = (
+            (f['gundlach_reckoning_norm'] > 0.5) & 
+            (f['prob_credit_collapse'] > 0.3)
+        ).astype(int)
+        
+        f['inequality_trap'] = (
+            (f['stevenson_inequality_norm'] > 0.6) & 
+            (f['prob_stagflation'] > 0.4)
+        ).astype(int)
+        
         f['tech_monopoly'] = (f['thiel_monopoly_norm'] > 0.6).astype(int)
+        
         f['geopolitical_shock'] = (f['geopolitical_risk_norm'] > 0.7).astype(int)
 
-        # Regime label
+        # Regime classification
         conditions = [
             f['debt_unsustainable'],
             f['tech_monopoly'],
@@ -218,9 +324,11 @@ class IntegratedTheoryFeatures:
         ]
         choices = ['CRISIS', 'TECH_MONOPOLY', 'INEQUALITY_TRAP', 'GEOPOLITICAL_SHOCK']
         f['regime'] = np.select(conditions, choices, default='TRANSITION')
+        
         return self
 
-    def build_features(self):
+    def build_all_features(self):
+        """Build complete feature set"""
         (self.dalio_forces()
          .stevenson_inequality()
          .thiel_monopoly()
@@ -240,8 +348,9 @@ def main():
 
     df = pd.read_csv(args.input, index_col=0, parse_dates=True)
     engine = IntegratedTheoryFeatures(df)
-    features = engine.build_features()
+    features = engine.build_all_features()
     features.to_csv(args.output)
+    print(f"✅ Features saved to {args.output}")
 
 
 if __name__ == "__main__":