data/data.py

import os
from datetime import datetime, timedelta
import ccxt
import pandas as pd
#import numpy as np
import calendar
#from scipy import signal
from scipy.signal import savgol_filter
from pykalman import KalmanFilter


class HistoricalDataFetcher:

  def __init__(self,
               exchange_name='coinbasepro',
               symbol='BTC/USD',
               timeframe='1d',
               start_date=datetime(2015, 7, 1),
               end_date=datetime.now()):
    self.exchange = getattr(ccxt, exchange_name)()
    self.symbol = symbol
    self.timeframe = timeframe
    self.start_date = start_date
    self.end_date = end_date
    self.raw_path = f"data/{self.symbol.replace('/','')}"
    self.clean_path = f"data/clean_{self.symbol.replace('/','')}"

    # Crear la carpeta para los archivos raw si no existe
    if not os.path.exists(self.raw_path):
      os.makedirs(self.raw_path)
    # Crear la carpeta para los archivos limpios si no existe
    if not os.path.exists(self.clean_path):
      os.makedirs(self.clean_path)

  def fetch_data(self):
    start_date = self.start_date
    end_date = self.end_date

    while start_date < end_date:

      year = start_date.year
      month = start_date.month

      # Determine the number of days in the month
      num_days = calendar.monthrange(year, month)[1]

      # Calculate the start and end dates for the month
      since = self.exchange.parse8601(
        start_date.strftime('%Y-%m-%d') + 'T00:00:00Z')
      until = self.exchange.parse8601(
        (start_date + timedelta(days=num_days)).strftime('%Y-%m-%d') +
        'T23:59:59Z')

      print(f"Fetching data for {calendar.month_name[month]} {year}")

      # Get historical data for the month
      ohlcv = self.exchange.fetch_ohlcv(self.symbol,
                                        self.timeframe,
                                        since,
                                        limit=None,
                                        params={'end': until})

      # Convert the data to a pandas DataFrame
      df = pd.DataFrame(
        ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])

      # Convert the timestamp to a datetime and set it as the index
      df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
      df.set_index('timestamp', inplace=True)

      # Save the data to a CSV file
      filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{start_date.strftime('%Y%m')}.csv"
      file_path = os.path.join(self.raw_path, filename)

      df.to_csv(file_path)

      print(f"Data saved to {filename}")

      # Increment the start date to the next month
      start_date = start_date.replace(day=1) + timedelta(days=32)
      start_date = start_date.replace(day=1)

  def get_data(self):
    # Obtener una lista de todos los archivos en la carpeta "data"
    data_files = os.listdir(self.raw_path)
    # Crear una lista vacía para almacenar los DataFrames de cada archivo
    data_frames = []

    # Cargar cada archivo CSV y agregar su DataFrame a la lista
    for file_name in data_files:
      if file_name.endswith(".csv"):
        file_path = os.path.join(self.raw_path, file_name)
        data = pd.read_csv(file_path, index_col=0, parse_dates=True)
        data_frames.append(data)

    # Concatenar los DataFrames en un solo DataFrame
    all_data = pd.concat(data_frames)
    return all_data

  def apply_filters(self,
                    data,
                    window_size=11,
                    polyorder=2,
                    kalman_observation_covariance=0.003,
                    kalman_transition_covariance=0.01,
                    kalman_initial_state_covariance=1000):

    # Aplicar suavizado exponencial
    alpha = 0.5
    data['close_ewm'] = data['close'].ewm(alpha=alpha, adjust=False).mean()

    # Aplicar filtro de Kalman
    kf = KalmanFilter(observation_covariance=kalman_observation_covariance,
                      transition_covariance=kalman_transition_covariance,
                      initial_state_covariance=kalman_initial_state_covariance,
                      n_dim_obs=1)
    data['close_kf'], _ = kf.filter(data['close'].values)

    # Aplicar suavizado de Savitzky-Golay
    data['close_savgol'] = savgol_filter(data['close'], window_size, polyorder)

    # Guardar los datos suavizados en un archivo CSV
    filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{self.start_date.strftime('%Y%m')}_cleaned.csv"
    file_path = os.path.join(self.clean_path, filename)

    data.to_csv(file_path)

    return data