import plost import streamlit as st import pandas as pd import numpy as np import jsonlines as jsl import datetime import altair as alt @st.experimental_memo def get_balance_data(): balance_data = {'usdt':[],'usdc':[],'usd':[],'dai':[],'eur':[],'chf':[],'gbp':[],'cad':[],'aud':[],'paxg':[],'timestamp':[]} with jsl.open('data/data_collection/balance_record_model.json') as reader: for obj in reader: if obj['balance_type'] == 'trade': continue balance_data['timestamp'].append(obj['record_time']['$date']) for key in balance_data.keys(): if key == 'timestamp': continue balance_data[key].append(obj['balances'][key]) balance_df = pd.DataFrame(balance_data) balance_df['timestamp'] = balance_df.timestamp.apply(lambda x:x.replace('T',' ').replace('Z','').split('.')[0]) balance_df['timestamp'] = pd.to_datetime(balance_df.timestamp) - datetime.timedelta(hours=8) keys = tuple(balance_data.keys())[:-1] return balance_df,keys @st.experimental_memo def get_trade_data(): trade_data = {'currency_pair':[],'price':[],'amount':[],'side':[],'time':[]} with jsl.open('data/data_collection/trade_record_model.json') as reader: for obj in reader: trade_data['currency_pair'].append(obj['currency_pair']) trade_data['price'].append(obj['price']) trade_data['amount'].append(obj['amount']) trade_data['side'].append(obj['side']) trade_data['time'].append(obj['trade_time']['$date']) trade_df = pd.DataFrame(trade_data) trade_df['time'] = trade_df.time.apply(lambda x:x.replace('T',' ').replace('Z','')) trade_df['time'] = pd.to_datetime(trade_df.time) - datetime.timedelta(hours=8) return trade_df @st.experimental_memo def get_ibkr_data(): ibkr_data = {'currency_pair':[],'price':[],'time':[]} with jsl.open('data/data_collection/history_tick_model.json') as reader: for obj in reader: try: ibkr_data['price'].append(float(obj['last'])) ibkr_data['currency_pair'].append(obj['symbol']) ibkr_data['time'].append(obj['time']['$date']) except: pass df_ibkr = pd.DataFrame(ibkr_data) df_ibkr['time'] = df_ibkr.time.apply(lambda x:x.replace('T',' ').replace('Z','')) df_ibkr['time'] = pd.to_datetime(df_ibkr.time) - datetime.timedelta(hours=8) df_ibkr.index = df_ibkr.time return df_ibkr @st.experimental_memo def get_predict_data(): vol_data = {"usdt_gbp":[],"usdc_usdt":[],"usdt_eur":[],"usdt_aud":[],"aud_usd":[],"usdt_chf":[],"usdc_usd":[],"eur_usd":[],"usd_cad":[],"dai_usdt":[],"usdc_aud":[],"paxg_usd":[],"dai_usd":[],"gbp_usd":[],"usdt_cad":[],"usd_chf":[],"usdt_usd":[],"usdc_eur":[],"usdc_gbp":[]} fp_data = {"eur":[],"gbp":[],"cad":[],"paxg":[],"chf":[],"aud":[]} vol_time = {"usdt_gbp":[],"usdc_usdt":[],"usdt_eur":[],"usdt_aud":[],"aud_usd":[],"usdt_chf":[],"usdc_usd":[],"eur_usd":[],"usd_cad":[],"dai_usdt":[],"usdc_aud":[],"paxg_usd":[],"dai_usd":[],"gbp_usd":[],"usdt_cad":[],"usd_chf":[],"usdt_usd":[],"usdc_eur":[],"usdc_gbp":[]} fp_time = {"eur":[],"gbp":[],"cad":[],"paxg":[],"chf":[],"aud":[]} with jsl.open('data/data_collection/predict_record_model.json') as reader: for obj in reader: if obj['predict_type'] == 'volatility_loss': for key in vol_data.keys(): if key not in obj['volatility_loss']: pass else: vol_data[key].append(obj['volatility_loss'][key]) vol_time[key].append(obj['record_time']['$date'].replace('T',' ').replace('Z','')) elif obj['predict_type'] == 'fp_premium': for key in fp_data.keys(): if key not in obj['fp_premium']: pass else: fp_data[key].append(obj['fp_premium'][key]) fp_time[key].append(obj['record_time']['$date'].replace('T',' ').replace('Z','')) # vol_df = pd.DataFrame(vol_data) # fp_df = pd.DataFrame(fp_data) vol_keys = vol_data.keys() fp_keys = fp_data.keys() return vol_data,fp_data,tuple(vol_keys),tuple(fp_keys),vol_time,fp_time @st.experimental_memo def get_ibkr_parquet(): df = pd.read_parquet('data/data_collection/ibkr_p.parquet.gzip') return df @st.experimental_memo def get_trade_parquet(): df = pd.read_parquet('data/data_collection/trade_p.parquet.gzip') return df @st.experimental_memo def fx_pair(trade_df,df_ibkr,trade_pair,ibkr_pair,fp_df=None,inv=False): trade_df.index = trade_df.time trade_df = trade_df.loc[str(df_ibkr.index[0]):str(df_ibkr.index[-1])] aud_trade = trade_df[trade_df.currency_pair==trade_pair] if fp_df is None: aud_ibkr = df_ibkr[df_ibkr.currency_pair==ibkr_pair][['price']] aud_ibkr = aud_ibkr.resample('1s').agg('last').ffill() else: aud_ibkr = fp_df[['fp']] #aud_trade['time'] = aud_trade.time.apply(lambda x:pd.to_datetime(x).tz_localize('UTC')) #aud_trade.index = aud_trade.time aud_trade.rename(columns={'price':'trade_price'},inplace=True) #aud_trade['time'] = aud_trade.index if fp_df is None: aud_trade['ibkr_price'] = aud_trade.time.apply(lambda x:aud_ibkr.loc[str(x).split('.')[0]].price) else: aud_trade['fp'] = aud_trade.time.apply(lambda x:aud_ibkr.loc[str(x).split('.')[0]].fp) composite = aud_trade if inv: composite['premium'] = composite.trade_price * (composite.ibkr_price) else: if fp_df is None: composite['premium'] = composite.trade_price * (1/composite.ibkr_price) else: composite['premium'] = composite.trade_price * (1/composite.fp) # trade_first = str(composite.index[0]) # trade_last = str(composite.index[-1]) # sns.set(rc={'figure.figsize':(70,20)}) # plt.subplot(211) # pa = ['r','g'] if len(composite.type.unique()) == 2 else [p[composite.type.unique()[0]]] # ax = sns.scatterplot(data=composite, x=composite.index, y="premium", size="vol",hue='type',sizes=(40, 300),palette=pa) # ax.axes.set_title(f'{trade_pair}_Trade_Vis_Official',fontsize=50) # plt.axhline(1) # #ax.axes.set_title(trade_pair+'_Trade_Vis',fontsize=50) # # ax.set_xlabel("premium",fontsize=30) # # ax.set_ylabel("Time",fontsize=20) # plt.subplot(212) # if fp_df is None: # if inv: # sns.lineplot(data=1/aud_ibkr[['price']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4) # else: # sns.lineplot(data=aud_ibkr[['price']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4) # else: # sns.lineplot(data=aud_ibkr[['fp']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4) # plt.savefig(f'{trade_pair}.jpg') return composite # @st.experimental_memo # def get_stable_composite(): # aud_trade = trade_df[trade_df.pair==pair] # #aud_trade['time'] = aud_trade.time.apply(lambda x:pd.to_datetime(x).tz_localize('UTC')) # #aud_trade.index = aud_trade.time # aud_trade.rename(columns={'price':'trade_price'},inplace=True) # ax = sns.scatterplot(data=aud_trade, x=aud_trade.index, y="trade_price", size="vol",hue='type',sizes=(40, 300),palette=['r','g']) # ax.axes.set_title(f'{pair}_Trade_Vis_Official',fontsize=30) # sns.set(rc={'figure.figsize':(70,20)}) # ax.set_xlabel("Time",fontsize=30) # ax.set_ylabel("Price",fontsize=20) def get_cross(name,df_ibkr): pair_a = '' pair_b = '' if name == 'eur_chf': pair_a = 'EURUSD' pair_b = 'CHFUSD' elif name == 'eur_gbp': pair_a = 'EURUSD' pair_b = 'GBPUSD' else: pair_a = 'XAUUSD' pair_b = 'EURUSD' df_a = df_ibkr[df_ibkr.currency_pair==pair_a].resample('1s').agg('last').ffill() df_b = df_ibkr[df_ibkr.currency_pair==pair_b].resample('1s').agg('last').ffill() composite = df_a.join(df_b,rsuffix='_b') composite['fp'] = composite['price']/composite['price_b'] return composite