File size: 4,557 Bytes
85a54f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | import os
from datetime import datetime, timedelta
import ccxt
import pandas as pd
#import numpy as np
import calendar
#from scipy import signal
from scipy.signal import savgol_filter
from pykalman import KalmanFilter
class HistoricalDataFetcher:
def __init__(self,
exchange_name='coinbasepro',
symbol='BTC/USD',
timeframe='1d',
start_date=datetime(2015, 7, 1),
end_date=datetime.now()):
self.exchange = getattr(ccxt, exchange_name)()
self.symbol = symbol
self.timeframe = timeframe
self.start_date = start_date
self.end_date = end_date
self.raw_path = f"data/{self.symbol.replace('/','')}"
self.clean_path = f"data/clean_{self.symbol.replace('/','')}"
# Crear la carpeta para los archivos raw si no existe
if not os.path.exists(self.raw_path):
os.makedirs(self.raw_path)
# Crear la carpeta para los archivos limpios si no existe
if not os.path.exists(self.clean_path):
os.makedirs(self.clean_path)
def fetch_data(self):
start_date = self.start_date
end_date = self.end_date
while start_date < end_date:
year = start_date.year
month = start_date.month
# Determine the number of days in the month
num_days = calendar.monthrange(year, month)[1]
# Calculate the start and end dates for the month
since = self.exchange.parse8601(
start_date.strftime('%Y-%m-%d') + 'T00:00:00Z')
until = self.exchange.parse8601(
(start_date + timedelta(days=num_days)).strftime('%Y-%m-%d') +
'T23:59:59Z')
print(f"Fetching data for {calendar.month_name[month]} {year}")
# Get historical data for the month
ohlcv = self.exchange.fetch_ohlcv(self.symbol,
self.timeframe,
since,
limit=None,
params={'end': until})
# Convert the data to a pandas DataFrame
df = pd.DataFrame(
ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume'])
# Convert the timestamp to a datetime and set it as the index
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
df.set_index('timestamp', inplace=True)
# Save the data to a CSV file
filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{start_date.strftime('%Y%m')}.csv"
file_path = os.path.join(self.raw_path, filename)
df.to_csv(file_path)
print(f"Data saved to {filename}")
# Increment the start date to the next month
start_date = start_date.replace(day=1) + timedelta(days=32)
start_date = start_date.replace(day=1)
def get_data(self):
# Obtener una lista de todos los archivos en la carpeta "data"
data_files = os.listdir(self.raw_path)
# Crear una lista vacía para almacenar los DataFrames de cada archivo
data_frames = []
# Cargar cada archivo CSV y agregar su DataFrame a la lista
for file_name in data_files:
if file_name.endswith(".csv"):
file_path = os.path.join(self.raw_path, file_name)
data = pd.read_csv(file_path, index_col=0, parse_dates=True)
data_frames.append(data)
# Concatenar los DataFrames en un solo DataFrame
all_data = pd.concat(data_frames)
return all_data
def apply_filters(self,
data,
window_size=11,
polyorder=2,
kalman_observation_covariance=0.003,
kalman_transition_covariance=0.01,
kalman_initial_state_covariance=1000):
# Aplicar suavizado exponencial
alpha = 0.5
data['close_ewm'] = data['close'].ewm(alpha=alpha, adjust=False).mean()
# Aplicar filtro de Kalman
kf = KalmanFilter(observation_covariance=kalman_observation_covariance,
transition_covariance=kalman_transition_covariance,
initial_state_covariance=kalman_initial_state_covariance,
n_dim_obs=1)
data['close_kf'], _ = kf.filter(data['close'].values)
# Aplicar suavizado de Savitzky-Golay
data['close_savgol'] = savgol_filter(data['close'], window_size, polyorder)
# Guardar los datos suavizados en un archivo CSV
filename = f"{self.symbol.replace('/','')}_{self.timeframe}_{self.start_date.strftime('%Y%m')}_cleaned.csv"
file_path = os.path.join(self.clean_path, filename)
data.to_csv(file_path)
return data
|