data_fetcher.py

import os
import pytz
import numpy as np
import pandas as pd
import ta
import MetaTrader5 as mt5
from datetime import datetime

def fetch_data():
    if not mt5.initialize():
        print(f"MT5 initialize() failed, error code: {mt5.last_error()}")
        return None

    symbol = "XAUUSDc"
    
    # Direct M3 fetch (Terminal was likely stalled before)
    timeframe = mt5.TIMEFRAME_M3
    
    # Use 2026 dates (updated methodologies)
    # Fetching from March 1st to current (approx 19th)
    utc_from = datetime(2026, 3, 1)
    utc_to = datetime(2026, 3, 19)
    
    print(f"Fetching data for {symbol} on M3 timeframe...")
    rates = mt5.copy_rates_range(symbol, timeframe, utc_from, utc_to)
    
    if rates is None or len(rates) == 0:
        print(f"Failed to fetch rates for {symbol}, error code: {mt5.last_error()}")
        mt5.shutdown()
        return None
        
    df = pd.DataFrame(rates)
    df['time'] = pd.to_datetime(df['time'], unit='s')
    df.set_index('time', inplace=True)
    
    # Ensure standard OHLCV names
    df = df[['open', 'high', 'low', 'close', 'tick_volume', 'spread', 'real_volume']]
    
    print(f"Fetched and resampled {len(df)} rows. Calculating 100+ vectorized indicators...")
    
    # Generate 90+ indicators automatically using the 'ta' library
    df = ta.add_all_ta_features(df, open="open", high="high", low="low", close="close", volume="tick_volume", fillna=True)
    
    # Add EXTRA indicators to exceed 100 count (approx 110-120 total)
    # 1. Rolling statistics (10-step windows)
    for col in ['open', 'high', 'low', 'close']:
        df[f'{col}_roll_mean_10'] = df[col].rolling(10).mean()
        df[f'{col}_roll_std_10'] = df[col].rolling(10).std()
        
    # 2. Lags (1-5)
    for i in range(1, 6):
        df[f'close_lag_{i}'] = df['close'].shift(i)
        
    # 3. RSI Variants
    df['rsi_fast'] = ta.momentum.rsi(df['close'], window=7)
    df['rsi_slow'] = ta.momentum.rsi(df['close'], window=21)
    
    # 4. Volatility Helpers
    df['atr_custom'] = ta.volatility.average_true_range(df['high'], df['low'], df['close'], window=14)
    
    # Drop rows with NaN from new indicators
    df.dropna(inplace=True)
    
    # Drop columns that have all NaNs
    df.dropna(axis=1, how='all', inplace=True)
    
    # Drop rows with any NaN values resulting from indicator lookbacks
    df.dropna(inplace=True)
    
    output_filename = "XAUUSD_M3_Data.csv"
    print(f"Saving {len(df)} clean rows to {output_filename} (Columns: {len(df.columns)})...")
    df.to_csv(output_filename)
    
    print("Data extraction and indicator generation complete!")
    mt5.shutdown()

if __name__ == "__main__":
    fetch_data()