File size: 5,454 Bytes
093b0a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | import datetime
import json
import os
from pathlib import Path
import torch
from utils.tools import dotdict
import pandas as pd
# Args / Settings helper functions
def args_from_setting(setting, args):
# pattern = r"(.+)_(.+)_ft(.+)_sl(.+)_ll(.+)_pl(.+)_ei(.+)_di(.+)_co(.+)_i(.+)_dm(.+)_nh(.+)_el(.+)_dl(.+)_df(.+)_at(.+)_fc(.+)_eb(.+)_dt(.+)_mx(.+)_(.+)_(.+).*"
# match = re.search(pattern, setting)
# if match:
# conv = lambda x: int(x) if x.isdigit() else (False if x=="False" else (True if x=="True" else x))
# (args.model, args.data, args.features,
# args.seq_len, args.label_len, args.pred_len,
# args.enc_in, args.dec_in, args.c_out, args.inverse,
# args.d_model, args.n_heads, args.e_layers, args.d_layers, args.d_ff, args.attn, args.factor,
# args.t_embed, args.distil, args.mix, args.des, ii) = map(conv, match.groups())
# print(args)
# else:
# raise Exception("Issue with setting name")
path = f"results/{setting}/args.json"
assert os.path.exists(path), f"{path}/args.json doesn't exist"
with open(path, "r") as f:
args = json.load(f)
return dotdict(args)
def setting_from_args(args, ii=0):
setting = "{}_{}_ft{}_sl{}_ll{}_pl{}_ei{}_di{}_co{}_i{}_dm{}_nh{}_el{}_dl{}_df{}_at{}_fc{}_eb{}_dt{}_mx{}_{}_{}".format(
args.model,
args.data,
args.features,
args.seq_len,
args.label_len,
args.pred_len,
args.enc_in,
args.dec_in,
args.c_out,
args.inverse,
args.d_model,
args.n_heads,
args.e_layers,
args.d_layers,
args.d_ff,
args.attn,
args.factor,
args.t_embed,
args.distil,
args.mix,
args.des,
ii,
)
return setting
def bbtest_setting(args):
time_label = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
setting = "{}_{}_sl{}_ei{}_dm{}_nh{}_el{}_eb{}".format(
time_label,
args.model,
args.seq_len,
args.enc_in,
args.d_model,
args.n_heads,
args.e_layers,
args.t_embed,
)
return setting
def write_df(data, out_file, append=""):
# Save flatten
og_cols = data.columns.copy()
data.columns = data.columns.to_flat_index()
data.columns = pd.Index(["_".join(col) for col in data.columns])
if append:
dot_loc = out_file.rfind(".")
out_file = f"{out_file[:dot_loc]}_{append}{out_file[dot_loc:]}"
if os.path.exists(out_file):
# Move current file to data/old
data_old = "data/old"
if not os.path.exists(data_old):
os.makedirs(data_old)
new_file_name = f"{out_file[:out_file.rfind('.')].replace('./','').replace('/','_')}_{datetime.datetime.now().strftime('%d_%m_%Y_%H_%M_%S')}{out_file[out_file.rfind('.'):]}"
os.rename(out_file, os.path.join(data_old, new_file_name))
else:
# Just attempt to make directories just incase
os.makedirs(Path(out_file).parent, exist_ok=True)
data.to_csv(out_file)
data.columns = og_cols
return out_file
# write_df(df, "test.csv")
def read_data(out_file="realdata.csv", stock=True):
data = pd.read_csv(out_file, index_col=0)
if not stock:
# Convert value timeseries into open close
converter = lambda col: f"{col}_open"
data.columns = data.columns.map(converter)
for column in data.columns:
data[f"{column.split('_')[0]}_close"] = data[column].shift(-1)
data = data.reindex(sorted(data.columns), axis=1)
converter = lambda col: tuple(col.split("_"))
# ast.literal_eval
data.columns = data.columns.map(converter)
data.index = pd.to_datetime(data.index)
if data.index.tz is None:
print("Warning: data did not have timestamp, adding utc")
data.index = pd.to_datetime(data.index, utc=True)
return data
def add_tz(data, time_zone="US/Eastern"):
"""Add timezone to timezone-unlabled df"""
t = pd.to_datetime(data.index).to_series()
data.index = t.dt.tz_localize(time_zone)
return data
def convert_tz(data, time_zone="US/Eastern"):
t = data.index.to_series()
t = t.dt.tz_convert(time_zone)
data.index = t
return data
# args.use_gpu = True if torch.cuda.is_available() else False
# args.gpu = 1
# args.use_multi_gpu = True
# args.devices = '0,1'
# if args.use_gpu and args.use_multi_gpu:
# args.devices = args.devices.replace(' ','')
# device_ids = args.devices.split(',')
# args.device_ids = [int(id_) for id_ in device_ids]
# args.gpu = args.device_ids[0]
def handle_gpu(args, gpu=None):
if not gpu and gpu is not None:
# Don't use gpu
args.use_gpu = False
args.use_multi_gpu = False
return
args.use_gpu = True if torch.cuda.is_available() else False
if not args.use_gpu:
return
if gpu is None:
# Use all gpus
c = torch.cuda.device_count()
args.device_ids = list(map(int, range(torch.cuda.device_count())))
args.devices = ",".join(map(str, args.device_ids))
else:
# Passed gpu(s)
gpu = str(gpu)
args.devices = gpu.replace(" ", "")
args.device_ids = [int(id_) for id_ in args.devices.split(",")]
if len(args.device_ids) >= 1:
args.use_multi_gpu = len(args.device_ids) > 1
args.gpu = int(args.device_ids[0])
|