Create geo_macro_df.py

geo_macro_df.py

@@ -0,0 +1,422 @@
+# geo_macro_df.py
+
+import pandas as pd
+import numpy as np
+import yfinance as yf
+import requests
+from datetime import datetime
+import warnings
+warnings.filterwarnings('ignore')
+
+# ======================
+# CONFIGURATION
+# ======================
+
+# Add your FRED API key here
+FRED_API_KEY = '23f3511b0ca43918ccd503ef64cb844e' 
+
+
+# ======================
+# UNIFIED DATA DOWNLOADER
+# ======================
+
+class UnifiedMarketDataDownloader:
+    """Downloads all market data into a single DataFrame"""
+    
+    def __init__(self, fred_api_key=None):
+        self.fred_api_key = fred_api_key
+        
+        # All market data sources
+        self.data_sources = {
+            # US Rates & Currencies
+            'DGS10': '^TNX', 
+            'DGS2': '^FVX', 
+            'DGS3MO': '^IRX',
+            'DXY': 'DX-Y.NYB', 
+            'EURUSD': 'EURUSD=X', 
+            'JPYUSD': 'JPYUSD=X',
+            
+            # US Equity Indices
+            'SP500': '^GSPC', 
+            'NASDAQ': '^IXIC', 
+            'RUSSELL': '^RUT',
+            'DJI': '^DJI', 
+            'VIX': '^VIX', 
+            'VXN': '^VXN',
+            
+            # Commodities
+            'Gold': 'GC=F', 
+            'Oil': 'CL=F', 
+            'Copper': 'HG=F',
+            'Silver': 'SI=F', 
+            'NaturalGas': 'NG=F',
+            
+            # Credit & Fixed Income
+            'HYG': 'HYG', 
+            'JNK': 'JNK', 
+            'LQD': 'LQD',
+            'TIP': 'TIP', 
+            'TLT': 'TLT',
+            
+            # Global Markets
+            'China': 'FXI',
+            'China_Tech': 'KWEB',
+            'Europe': 'FEZ',
+            'Europe_Financials': 'EUFN',
+            'Japan': 'EWJ',
+            'South_Korea': 'EWY',
+            'Taiwan': 'EWT',
+            'India': 'INDA',
+            'Brazil': 'EWZ',
+            'Emerging_Markets': 'EEM',
+            
+            # Global Currencies
+            'CNY': 'CNY=X',
+            'JPY': 'JPY=X',
+            'EUR': 'EUR=X',
+            'GBP': 'GBP=X',
+            
+            # Geopolitical Indicators
+            'Defense_Stocks': 'ITA',
+            'Cybersecurity': 'HACK',
+            'Energy_Security': 'XLE',
+            'Gold_Safe_Haven': 'GLD',
+            'US_Treasuries_Long': 'TLT',
+            
+            # Sectors
+            'Technology': 'XLK',
+            'Financials': 'XLF',
+            'Healthcare': 'XLV',
+            'Consumer_Discretionary': 'XLY',
+            'Consumer_Staples': 'XLP',
+            'Energy': 'XLE',
+            'Materials': 'XLB',
+            'Industrials': 'XLI',
+            'Utilities': 'XLU',
+            'Real_Estate': 'XLRE',
+            'Communication_Services': 'XLC',
+            
+            # Sector Details
+            'Regional_Banks': 'KRE',
+            'Homebuilders': 'XHB',
+            'Retail': 'XRT',
+            'Transportation': 'XTN',
+            'Semiconductors': 'SMH',
+            'Clean_Energy': 'ICLN',
+            'Aerospace_Defense': 'XAR',
+            
+            # Supply Chain & Logistics
+            'Baltic_Dry_Index': 'BDRY',  # Shipping costs proxy
+            'Logistics': 'XTN',
+            
+            # Credit Spreads & Risk
+            'Investment_Grade_Spread': 'LQD',  # Already have, key for credit risk
+            'Emerging_Market_Debt': 'EMB',
+            'Muni_Bonds': 'MUB',
+            
+            # Inflation Breakevens
+            'Inflation_Protected': 'TIP',  # Already have
+            'Short_Term_Treasuries': 'SHY',
+            'Intermediate_Treasuries': 'IEF',
+            
+            # Currency Volatility
+            'USD_Emerging': 'UUP',  # USD strength
+            'Gold_Miners': 'GDX',  # Gold mining companies (more volatile than GLD)
+            
+            # Economic Cycle Indicators
+            'Small_Cap_Value': 'IWN',  # Early cycle indicator
+            'High_Dividend': 'VYM',  # Late cycle/defensive
+            'Growth_Stocks': 'VUG',  # Risk-on
+            'Value_Stocks': 'VTV',  # Risk-off rotation
+            
+            # Liquidity & Credit Conditions
+            'Mortgage_REITs': 'REM',  # Interest rate sensitivity
+            'Preferred_Stock': 'PFF',  # Credit conditions
+            
+            # Global Safe Havens
+            'Swiss_Franc': 'CHF=X',
+            'Gold_Futures': 'GC=F',  # Already have as 'Gold'
+            'Bitcoin': 'BTC-USD',  # Alternative safe haven / risk asset
+            
+            # Commodity Inflation
+            'Agricultural': 'DBA',
+            'Base_Metals': 'DBB',
+            'Crude_Oil': 'USO',
+            
+            # Labor Market
+            'Staffing': 'SIA',  # Staffing index (leading indicator)
+            
+            # Housing Market
+            'Mortgage_Backed_Securities': 'MBB',
+            'REITs': 'VNQ',
+            
+            # Consumer Health
+            'Consumer_Discretionary_vs_Staples': 'XLY',  # Already have
+            'Restaurants': 'EAT',  # Consumer spending proxy
+            'Retail_Luxury': 'RL',  # High-end consumer
+            
+            # Tech Innovation Cycles
+            'Cloud_Computing': 'SKYY',
+            'Robotics_AI': 'BOTZ',
+            'Fintech': 'FINX',
+            
+            # Geopolitical Tension Proxies
+            'Uranium': 'URA',  # Nuclear/energy security
+            'Rare_Earth': 'REMX',  # Supply chain geopolitics
+            'Water': 'PHO',  # Resource scarcity
+            
+            # Sentiment & Positioning
+            'Leveraged_Loans': 'BKLN',  # Credit appetite
+            'TIPS_Spread': 'TIP',  # Inflation expectations
+        }
+        
+        # FRED Economic Series
+        self.fred_series = {
+            # Labor Market
+            'UNRATE': 'UNRATE',           # Unemployment Rate
+            'PAYEMS': 'PAYEMS',           # Non-Farm Payrolls
+            'ICSA': 'ICSA',               # Initial Jobless Claims
+            'JTSJOL': 'JTSJOL',           # Job Openings (JOLTS)
+            
+            # Inflation
+            'CPIAUCSL': 'CPIAUCSL',       # CPI
+            'CPILFESL': 'CPILFESL',       # Core CPI
+            'PPIACO': 'PPIACO',           # PPI
+            'PCEPILFE': 'PCEPILFE',       # Core PCE (Fed's preferred)
+            
+            # Production & Manufacturing
+            'INDPRO': 'INDPRO',           # Industrial Production
+            'IPMAN': 'IPMAN',             # Manufacturing Production
+            'TOTALSA': 'TOTALSA',         # Total Vehicle Sales
+            'UMTMVS': 'UMTMVS',           # Manufacturing New Orders
+            
+            # Money & Credit
+            'M2': 'M2SL',                 # M2 Money Supply
+            'WALCL': 'WALCL',             # Fed Balance Sheet
+            'TOTCI': 'TOTCI',             # Commercial & Industrial Loans
+            
+            # Consumer
+            'CONSUMER_SENTIMENT': 'UMCSENT',  # Consumer Sentiment
+            'RSXFS': 'RSXFS',             # Retail Sales
+            'PCE': 'PCE',                 # Personal Consumption
+            'PSAVERT': 'PSAVERT',         # Personal Saving Rate
+            
+            # Housing
+            'HOUST': 'HOUST',             # Housing Starts
+            'MORTGAGE30US': 'MORTGAGE30US',  # 30-Year Mortgage Rate
+            'CSUSHPISA': 'CSUSHPISA',     # Case-Shiller Home Price Index
+            
+            # Trade & GDP
+            'GDP': 'GDP',                 # GDP
+            'NETEXP': 'NETEXP',           # Net Exports
+            'BOPGSTB': 'BOPGSTB',         # Trade Balance
+            
+            # Yield Curve & Credit
+            'T10Y2Y': 'T10Y2Y',           # 10Y-2Y Yield Spread (recession indicator)
+            'T10YIE': 'T10YIE',           # 10Y Breakeven Inflation Rate
+            'BAMLH0A0HYM2': 'BAMLH0A0HYM2',  # High Yield Spread
+            'DFII10': 'DFII10',           # 10-Year TIPS
+            
+            # Leading Indicators
+            'USSLIND': 'USSLIND',         # Leading Index
+            'DCOILWTICO': 'DCOILWTICO',   # WTI Crude Oil Price
+            
+            # Corporate & Business
+            'CPROFIT': 'CPROFIT',         # Corporate Profits
+            'BUSLOANS': 'BUSLOANS',       # Business Loans
+        }
+    
+    def download_all_data(self, start_date='2018-01-01', end_date=None):
+        """
+        Download all market data and return a single unified DataFrame
+        
+        Parameters:
+        -----------
+        start_date : str
+            Start date in 'YYYY-MM-DD' format
+        end_date : str, optional
+            End date in 'YYYY-MM-DD' format (defaults to today)
+            
+        Returns:
+        --------
+        pd.DataFrame
+            Unified DataFrame with all market data, date-indexed
+        """
+        if end_date is None:
+            end_date = datetime.now().strftime('%Y-%m-%d')
+        
+        print("=" * 80)
+        print("🚀 UNIFIED MARKET DATA DOWNLOAD")
+        print("=" * 80)
+        print(f"📅 Period: {start_date} to {end_date}\n")
+        
+        # Step 1: Download all market data
+        print("📊 Downloading Market Data...")
+        market_data = self._download_market_data(start_date, end_date)
+        
+        # Step 2: Download FRED economic data
+        if self.fred_api_key and self.fred_api_key != 'your_api_key_here':
+            print("\n📈 Downloading Economic Data (FRED)...")
+            economic_data = self._download_fred_data(start_date, end_date)
+        else:
+            print("\n⚠️  FRED API key not provided - skipping economic data")
+            print("   Get your free API key at: https://fred.stlouisfed.org/docs/api/api_key.html")
+            economic_data = {}
+        
+        # Step 3: Merge everything into single DataFrame
+        print("\n🔗 Merging all data sources...")
+        unified_df = self._merge_all_data(market_data, economic_data, start_date, end_date)
+        
+        # Print summary
+        print("\n" + "=" * 80)
+        print("✅ DOWNLOAD COMPLETE")
+        print("=" * 80)
+        print(f"📊 Total Columns: {unified_df.shape[1]}")
+        print(f"📅 Total Rows: {unified_df.shape[0]}")
+        print(f"🗓️  Date Range: {unified_df.index.min().strftime('%Y-%m-%d')} to {unified_df.index.max().strftime('%Y-%m-%d')}")
+        print(f"📉 Missing Values: {unified_df.isnull().sum().sum()}")
+        print(f"✨ Completeness: {(1 - unified_df.isnull().sum().sum()/(unified_df.shape[0]*unified_df.shape[1]))*100:.2f}%")
+        
+        print("\n📋 Column Preview:")
+        print(unified_df.columns.tolist()[:10], "...\n")
+        
+        return unified_df
+    
+    def _download_market_data(self, start_date, end_date):
+        """Download all market data from Yahoo Finance"""
+        all_tickers = list(self.data_sources.values())
+        
+        print(f"   Downloading {len(all_tickers)} tickers...")
+        
+        # Download all at once (faster)
+        try:
+            data = yf.download(all_tickers, start=start_date, end=end_date, 
+                             progress=False, auto_adjust=True, threads=True)
+            
+            # Extract Close prices
+            if isinstance(data.columns, pd.MultiIndex):
+                close_data = data['Close']
+            else:
+                close_data = data[['Close']] if 'Close' in data.columns else data
+            
+            # Rename columns to our names
+            ticker_to_name = {v: k for k, v in self.data_sources.items()}
+            close_data.columns = [ticker_to_name.get(col, col) for col in close_data.columns]
+            
+            # Remove any columns that failed to download
+            close_data = close_data.dropna(axis=1, how='all')
+            
+            print(f"   ✅ Successfully downloaded {len(close_data.columns)} series")
+            return close_data
+            
+        except Exception as e:
+            print(f"   ⚠️  Batch download failed: {e}")
+            print("   Trying individual downloads...")
+            return self._download_individual(start_date, end_date)
+    
+    def _download_individual(self, start_date, end_date):
+        """Fallback: download tickers individually"""
+        data_dict = {}
+        
+        for name, ticker in self.data_sources.items():
+            try:
+                df = yf.download(ticker, start=start_date, end=end_date, 
+                               progress=False, auto_adjust=True)
+                
+                if not df.empty and 'Close' in df.columns:
+                    series = df['Close'].squeeze()
+                    if isinstance(series, pd.DataFrame):
+                        series = series.iloc[:, 0]
+                    if isinstance(series, pd.Series) and len(series) > 10:
+                        data_dict[name] = series
+                        print(f"   ✅ {name}")
+            except Exception as e:
+                print(f"   ❌ {name}: {str(e)[:50]}")
+                continue
+        
+        return pd.DataFrame(data_dict)
+    
+    def _download_fred_data(self, start_date, end_date):
+        """Download economic data from FRED"""
+        economic_data = {}
+        
+        for name, series_id in self.fred_series.items():
+            try:
+                url = "https://api.stlouisfed.org/fred/series/observations"
+                params = {
+                    'series_id': series_id,
+                    'api_key': self.fred_api_key,
+                    'file_type': 'json',
+                    'observation_start': start_date,
+                    'observation_end': end_date,
+                }
+                
+                response = requests.get(url, params=params, timeout=30)
+                
+                if response.status_code == 200:
+                    data = response.json()
+                    if 'observations' in data and data['observations']:
+                        df = pd.DataFrame(data['observations'])
+                        df['value'] = pd.to_numeric(df['value'], errors='coerce')
+                        df['date'] = pd.to_datetime(df['date'])
+                        series = df.set_index('date')['value'].dropna()
+                        
+                        if len(series) > 10:
+                            economic_data[name] = series
+                            print(f"   ✅ {name}")
+                else:
+                    print(f"   ❌ {name}: HTTP {response.status_code}")
+                    
+            except Exception as e:
+                print(f"   ❌ {name}: {str(e)[:50]}")
+                continue
+        
+        return economic_data
+    
+    def _merge_all_data(self, market_data, economic_data, start_date, end_date):
+        """Merge all data sources into single DataFrame with proper filling"""
+        
+        # Create base date range (business days)
+        date_range = pd.date_range(start=start_date, end=end_date, freq='D')
+        
+        # Start with market data
+        unified = market_data.copy()
+        unified = unified.reindex(date_range)
+        
+        # Add economic data
+        for name, series in economic_data.items():
+            unified[name] = series.reindex(date_range)
+        
+        # Forward fill then backward fill (handles weekends/holidays)
+        unified = unified.ffill().bfill()
+        
+        # Drop any columns that are still all NaN
+        unified = unified.dropna(axis=1, how='all')
+        
+        return unified
+
+
+
+'''
+
+# ======================
+# USAGE EXAMPLE
+# ======================
+
+if __name__ == "__main__":
+    # Initialize downloader
+    downloader = UnifiedMarketDataDownloader(fred_api_key=FRED_API_KEY)
+    
+    # Download all data
+    raw_market_data = downloader.download_all_data(
+        start_date='2018-01-01',
+        end_date=None  # defaults to today
+    )
+    
+
+    
+    # Save to CSV (optional)
+    # raw_market_data.to_csv('unified_market_data.csv')
+    # print("\n💾 Saved to: unified_market_data.csv")
+
+'''