utils.py

import plost
import streamlit as st
import pandas as pd
import numpy as np
import jsonlines as jsl
import datetime
import altair as alt

@st.experimental_memo
def get_balance_data():
    balance_data = {'usdt':[],'usdc':[],'usd':[],'dai':[],'eur':[],'chf':[],'gbp':[],'cad':[],'aud':[],'paxg':[],'timestamp':[]}
    with jsl.open('data/data_collection/balance_record_model.json') as reader:
        for obj in reader:
            if obj['balance_type'] == 'trade':
                continue
            balance_data['timestamp'].append(obj['record_time']['$date'])
            for key in balance_data.keys():
                if key == 'timestamp':
                    continue
                balance_data[key].append(obj['balances'][key])

    balance_df = pd.DataFrame(balance_data)
    balance_df['timestamp'] = balance_df.timestamp.apply(lambda x:x.replace('T',' ').replace('Z','').split('.')[0])
    balance_df['timestamp'] = pd.to_datetime(balance_df.timestamp) - datetime.timedelta(hours=8)
    keys = tuple(balance_data.keys())[:-1]
    return balance_df,keys



@st.experimental_memo
def get_trade_data():
    trade_data = {'currency_pair':[],'price':[],'amount':[],'side':[],'time':[]}
    with jsl.open('data/data_collection/trade_record_model.json') as reader:
        for obj in reader:
            trade_data['currency_pair'].append(obj['currency_pair'])
            trade_data['price'].append(obj['price'])
            trade_data['amount'].append(obj['amount'])
            trade_data['side'].append(obj['side'])
            trade_data['time'].append(obj['trade_time']['$date'])
    
    trade_df = pd.DataFrame(trade_data)
    trade_df['time'] = trade_df.time.apply(lambda x:x.replace('T',' ').replace('Z',''))
    trade_df['time'] = pd.to_datetime(trade_df.time) - datetime.timedelta(hours=8)
    return trade_df


@st.experimental_memo
def get_ibkr_data():
    ibkr_data = {'currency_pair':[],'price':[],'time':[]}
    with jsl.open('data/data_collection/history_tick_model.json') as reader:
        for obj in reader:
            try:
                ibkr_data['price'].append(float(obj['last']))
                ibkr_data['currency_pair'].append(obj['symbol'])
                ibkr_data['time'].append(obj['time']['$date'])
                
                
            except:
                pass
    
    df_ibkr = pd.DataFrame(ibkr_data)
    df_ibkr['time'] = df_ibkr.time.apply(lambda x:x.replace('T',' ').replace('Z',''))
    df_ibkr['time'] = pd.to_datetime(df_ibkr.time) - datetime.timedelta(hours=8)
    df_ibkr.index = df_ibkr.time

    return df_ibkr


@st.experimental_memo
def get_predict_data():

    vol_data = {"usdt_gbp":[],"usdc_usdt":[],"usdt_eur":[],"usdt_aud":[],"aud_usd":[],"usdt_chf":[],"usdc_usd":[],"eur_usd":[],"usd_cad":[],"dai_usdt":[],"usdc_aud":[],"paxg_usd":[],"dai_usd":[],"gbp_usd":[],"usdt_cad":[],"usd_chf":[],"usdt_usd":[],"usdc_eur":[],"usdc_gbp":[]}
    fp_data = {"eur":[],"gbp":[],"cad":[],"paxg":[],"chf":[],"aud":[]}
    vol_time = {"usdt_gbp":[],"usdc_usdt":[],"usdt_eur":[],"usdt_aud":[],"aud_usd":[],"usdt_chf":[],"usdc_usd":[],"eur_usd":[],"usd_cad":[],"dai_usdt":[],"usdc_aud":[],"paxg_usd":[],"dai_usd":[],"gbp_usd":[],"usdt_cad":[],"usd_chf":[],"usdt_usd":[],"usdc_eur":[],"usdc_gbp":[]}
    fp_time = {"eur":[],"gbp":[],"cad":[],"paxg":[],"chf":[],"aud":[]} 
    with jsl.open('data/data_collection/predict_record_model.json') as reader:
        for obj in reader:
            if obj['predict_type'] == 'volatility_loss':
                for key in vol_data.keys():
                    if key not in obj['volatility_loss']:
                        pass
                    else: 
                        vol_data[key].append(obj['volatility_loss'][key])
                        vol_time[key].append(obj['record_time']['$date'].replace('T',' ').replace('Z',''))
                
            elif obj['predict_type'] == 'fp_premium':
                for key in fp_data.keys():
                    if key not in obj['fp_premium']:
                        pass
                    else:
                        fp_data[key].append(obj['fp_premium'][key])
                        fp_time[key].append(obj['record_time']['$date'].replace('T',' ').replace('Z',''))
                

    # vol_df = pd.DataFrame(vol_data)
    # fp_df = pd.DataFrame(fp_data)
    
    vol_keys = vol_data.keys()
    fp_keys = fp_data.keys()
    return vol_data,fp_data,tuple(vol_keys),tuple(fp_keys),vol_time,fp_time



@st.experimental_memo
def get_ibkr_parquet():
    df = pd.read_parquet('data/data_collection/ibkr_p.parquet.gzip')
    return df
@st.experimental_memo
def get_trade_parquet():
    df = pd.read_parquet('data/data_collection/trade_p.parquet.gzip')
    return df


@st.experimental_memo
def fx_pair(trade_df,df_ibkr,trade_pair,ibkr_pair,fp_df=None,inv=False):
    trade_df.index = trade_df.time
    trade_df = trade_df.loc[str(df_ibkr.index[0]):str(df_ibkr.index[-1])]
    aud_trade = trade_df[trade_df.currency_pair==trade_pair]

    if fp_df is None:
        aud_ibkr = df_ibkr[df_ibkr.currency_pair==ibkr_pair][['price']]
        aud_ibkr = aud_ibkr.resample('1s').agg('last').ffill()
    else:
        aud_ibkr = fp_df[['fp']]


    #aud_trade['time'] = aud_trade.time.apply(lambda x:pd.to_datetime(x).tz_localize('UTC'))
    #aud_trade.index = aud_trade.time
    aud_trade.rename(columns={'price':'trade_price'},inplace=True)
    #aud_trade['time'] = aud_trade.index
    if fp_df is None:
        aud_trade['ibkr_price'] = aud_trade.time.apply(lambda x:aud_ibkr.loc[str(x).split('.')[0]].price)
    else:
        aud_trade['fp'] = aud_trade.time.apply(lambda x:aud_ibkr.loc[str(x).split('.')[0]].fp)
    composite = aud_trade
    
    
    if inv:
        composite['premium'] = composite.trade_price * (composite.ibkr_price)
    else:
        if fp_df is None:
            composite['premium'] = composite.trade_price * (1/composite.ibkr_price)
        else:
            composite['premium'] = composite.trade_price * (1/composite.fp) 
    # trade_first = str(composite.index[0])
    # trade_last = str(composite.index[-1])
    # sns.set(rc={'figure.figsize':(70,20)})
    # plt.subplot(211)
    # pa = ['r','g'] if len(composite.type.unique()) == 2 else [p[composite.type.unique()[0]]]
    # ax = sns.scatterplot(data=composite, x=composite.index, y="premium", size="vol",hue='type',sizes=(40, 300),palette=pa)
    # ax.axes.set_title(f'{trade_pair}_Trade_Vis_Official',fontsize=50)
    # plt.axhline(1)
    # #ax.axes.set_title(trade_pair+'_Trade_Vis',fontsize=50)
    # # ax.set_xlabel("premium",fontsize=30)
    # # ax.set_ylabel("Time",fontsize=20)
    # plt.subplot(212)
    # if fp_df is None:
    #     if inv:
    #         sns.lineplot(data=1/aud_ibkr[['price']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4)
    #     else:
    #         sns.lineplot(data=aud_ibkr[['price']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4) 
    # else:
    #     sns.lineplot(data=aud_ibkr[['fp']].loc[trade_first:trade_last].rename(columns={'price':'ibkr'}),palette=['b'],alpha=0.4)
    
    # plt.savefig(f'{trade_pair}.jpg')
    return composite

# @st.experimental_memo
# def get_stable_composite():
#     aud_trade = trade_df[trade_df.pair==pair]
#     #aud_trade['time'] = aud_trade.time.apply(lambda x:pd.to_datetime(x).tz_localize('UTC'))
#     #aud_trade.index = aud_trade.time
#     aud_trade.rename(columns={'price':'trade_price'},inplace=True)
    
#     ax = sns.scatterplot(data=aud_trade, x=aud_trade.index, y="trade_price", size="vol",hue='type',sizes=(40, 300),palette=['r','g'])
#     ax.axes.set_title(f'{pair}_Trade_Vis_Official',fontsize=30)
#     sns.set(rc={'figure.figsize':(70,20)})
#     ax.set_xlabel("Time",fontsize=30)
#     ax.set_ylabel("Price",fontsize=20)

def get_cross(name,df_ibkr):
    pair_a = ''
    pair_b = ''
    if name == 'eur_chf':
        pair_a = 'EURUSD'
        pair_b = 'CHFUSD'
    elif name == 'eur_gbp':
        pair_a = 'EURUSD'
        pair_b = 'GBPUSD'
    else:
        pair_a = 'XAUUSD'
        pair_b = 'EURUSD'


    df_a = df_ibkr[df_ibkr.currency_pair==pair_a].resample('1s').agg('last').ffill()
    df_b = df_ibkr[df_ibkr.currency_pair==pair_b].resample('1s').agg('last').ffill()
    composite = df_a.join(df_b,rsuffix='_b')
    composite['fp'] = composite['price']/composite['price_b']
    return composite