Duplicate from yifanxie/numerdash

Co-authored-by: Yifan Xie <yifanxie@users.noreply.huggingface.co>

.streamlit/config.toml

@@ -0,0 +1,2 @@
+[theme]
+base = "dark"

README.md

@@ -0,0 +1,8 @@
+---
+title: NumerDash
+emoji: 🚀🔥
+sdk: streamlit
+app_file: numerdash_app.py
+pinned: false
+duplicated_from: yifanxie/numerdash
+---

default_models.json

@@ -0,0 +1,6 @@
+{
+  "benchmark": [
+    "integration_test",
+    "integration_test_7"
+  ]
+}

numerdash_app.py

@@ -0,0 +1,975 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import os
+import sys
+sys.path.append(os.path.dirname(os.getcwd()))
+from project_tools import project_utils, project_config, numerapi_utils
+import warnings
+import plotly.express as px
+import json
+warnings.filterwarnings("ignore")
+from PIL import Image
+import plotly.express as px
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from streamlit import caching
+import time
+import traceback
+import datetime
+
+st.set_page_config(layout='wide')
+get_benchmark_data = True
+
+# get_dailyscore = True
+
+
+
+
+def sidebar_data_picker():
+    st.sidebar.subheader('Model Data Picker')
+    top_lb = st.sidebar.checkbox('top LB by corr', value=True)
+    top_tp3m = st.sidebar.checkbox('most profitable 3 month', value=True)
+    top_tp1y = st.sidebar.checkbox('most profitable 1 year', value=True)
+    special_list = st.sidebar.checkbox('model from specific users', value=True)
+    return top_lb, top_tp3m, top_tp1y, special_list
+
+
+# to be removed
+def model_data_picker_bak(values = None):
+    if values is None:
+        values = [True, True, True, True, True, True]
+    model_dict = {}
+    st.sidebar.subheader('Model Data Picker')
+    # top_lb = st.sidebar.checkbox('top LB by corr', value=values[0])
+    # top_tp3m = st.sidebar.checkbox('most profitable 3 month', value=values[1])
+    top_tp1y = st.sidebar.checkbox('most profitable 1 year', value=values[2])
+    special_list = st.sidebar.checkbox('model from specific users', value=values[3])
+    benchmark_list = st.sidebar.checkbox('benchmark models', value=values[4])
+    default_list = st.sidebar.checkbox('default models', value=values[5])
+    # if top_lb:
+    #     model_dict['top_corr'] = project_config.TOP_LB
+    # if top_tp3m:
+        # model_dict['top_3m'] = project_config.TP3M
+    if top_tp1y:
+        model_dict['top_1y'] = project_config.TP1Y
+    if benchmark_list:
+        model_dict['benchmark'] = project_config.BENCHMARK_MODELS
+    if special_list:
+        model_dict['iaai'] = project_config.IAAI_MODELS
+        # model_dict['arbitrage'] = project_config.ARBITRAGE_MODELS
+        # model_dict['mm'] = project_config.MM_MODELS
+        # model_dict['restrade'] = project_config.RESTRADE_MODELS
+
+    if default_list:
+        model_dict['yx'] = project_config.MODEL_NAMES + project_config.NEW_MODEL_NAMES
+        model_dict['mcv'] = project_config.MCV_MODELS + project_config.MCV_NEW_MODELS
+    return model_dict
+
+
+# to be removed
+def model_fast_picker_bak(models):
+    text_content = '''
+                    fast model picker by CSV string.
+                    example: "model1, model2, model3" 
+                   '''
+    text = st.sidebar.text_area(text_content)
+    result_models = []
+    if len(text)>0:
+        csv_parts = text.split(',')
+        for s in csv_parts:
+            m = s.strip()
+            if m in models:
+                result_models.append(m)
+    return list(dict.fromkeys(result_models))
+
+
+
+def default_model_picker():
+    picked_models = {}
+    if os.path.isfile('default_models.json'):
+        default_models_dict = project_utils.load_json('default_models.json')
+        for key in default_models_dict.keys():
+            picked_models[key] = default_models_dict[key]
+    if os.path.isfile('user_models.json'):
+        user_models_dict = project_utils.load_json('user_models.json')
+        for key in user_models_dict.keys():
+            picked_models[key] = user_models_dict[key]
+    return picked_models
+
+
+def model_fast_picker(model_list):
+    text_content = '''
+                    fast model picker by CSV string.
+                    example: "model1, model2, model3" 
+                   '''
+    text = st.sidebar.text_area(text_content)
+    result_models = []
+    if len(text)>0:
+        csv_parts = text.split(',')
+        for s in csv_parts:
+            m = s.strip()
+            if (m  in model_list): #and (m not in preselected_models):
+                result_models.append(m)
+    return list(dict.fromkeys(result_models))
+
+
+
+
+
+
+def generate_round_table(data, row_cts, c, r, sortcol='corrmmc'):
+    # rounds = data
+    # row_cts[c].write(2*r+c)
+    latest_round = int(data['roundNumber'].max())
+    earliest_round =  int(data['roundNumber'].min())
+    suggest_round = int(latest_round - (2*r+c))
+    select_round = row_cts[c].slider('select a round', earliest_round, latest_round, suggest_round, 1)
+    # row_cts[c].write(select_round)
+    round_data = data[data['roundNumber']==select_round].sort_values(by=sortcol, ascending=False).reset_index(drop=True)
+    round_resolved_time = round_data['roundResolveTime'][0]
+    # round_data = round_data[round_data['model'].isin(models)].reset_index(drop=True)
+    # latest_date = round_data['date'].values[0]
+    row_cts[c].write(f'round: {select_round}  resolved time: {round_resolved_time}')
+    row_cts[c].dataframe(round_data.drop(['roundNumber', 'roundResolveTime'], axis=1), height=max_table_height-100)
+
+
+
+
+
+
+def generate_dailyscore_metrics(data, row_cts, c, r):
+    # row_cts[c].write([r, c, 2*r+c])
+    select_metric = row_cts[c].selectbox("", list(id_metric_opt.keys()), index=2*r+c, format_func=lambda x: id_metric_opt[x])
+    latest_round = int(data['roundNumber'].max())
+    earliest_round = int(data['roundNumber'].min())
+    score = id_metric_score_dic[select_metric]
+    df = project_utils.calculate_rounddailysharpe_dashboard(data, latest_round, earliest_round, score).sort_values(by='sos', ascending=False)
+    row_cts[c].dataframe(df, height=max_table_height-100)
+    pass
+
+def get_roundmetric_data(data):
+    numfeats1 = ['corr', 'mmc', 'tc', 'corrmmc', 'corrtc', 'fncV3', 'fncV3_pct']
+    stat1 = ['sum', 'mean', 'count',
+             {'sharpe': project_utils.get_array_sharpe}]  # {'ptp':np.ptp}]#{'sharp':project_utils.get_array_sharpe}]
+    numfeats2 = ['corr_pct', 'mmc_pct', 'tc_pct','corrtc_avg_pct', 'corrmmc_avg_pct']
+    stat2 = ['mean']#, {'sharp': project_utils.get_array_sharpe}]
+
+    roundmetric_agg_rcp = [
+        [['model'], numfeats1, stat1],
+        [['model'], numfeats2, stat2]
+    ]
+
+    res = project_utils.groupby_agg_execution(roundmetric_agg_rcp, data)['model']
+    rename_dict = {}
+    for c in res.columns.tolist():
+        if c != 'model':
+            rename_dict[c] = c[6:] # remove 'model_' in column name
+    res.rename(columns = rename_dict, inplace=True)
+    return res
+
+
+def generate_round_metrics(data, row_cts, c, r):
+    select_metric = row_cts[c].selectbox("", list(roundmetric_opt.keys()), index=2*r+c, format_func=lambda x: roundmetric_opt[x])
+    cols = ['model']
+    # st.write(select_metric)
+    # st.write(data.columns.tolist())
+    for col in data.columns.tolist():
+        if select_metric =='corrmmc':
+            if (f'{select_metric}_' in col) or ('corrmmc_avg_' in col):
+                cols += [col]
+        elif select_metric =='corrtc':
+            if (f'{select_metric}_' in col) or ('corrtc_avg_' in col):
+                cols += [col]
+        else:
+            # if (f'{select_metric}_' in col) and (not('corrmmc' in col)) and (not('corrtc' in col)):
+            if (f'{select_metric}_' in col):
+                cols+= [col]
+
+    if select_metric != 'pct':
+        sort_col = select_metric+'_sharpe'
+    else:
+        sort_col = 'corr_pct_mean'
+    view_data = data[cols].sort_values(by=sort_col, ascending=False)
+    row_cts[c].dataframe(view_data)
+    pass
+
+
+def dailyscore_chart(data, row_cts, c, r, select_metric):
+    latest_round = int(data['roundNumber'].max())
+    earliest_round =  int(data['roundNumber'].min())
+    suggest_round = int(latest_round - (2*r+c))
+    select_round = row_cts[c].slider('select a round', earliest_round, latest_round, suggest_round, 1)
+    data = data[data['roundNumber']==select_round]
+    if len(data)>0:
+        fig = chart_pxline(data, 'date', y=select_metric, color='model', hover_data=list(histtrend_opt.keys()))
+        row_cts[c].plotly_chart(fig, use_container_width=True)
+    else:
+        row_cts[c].info('no data was found for the selected round')
+    pass
+
+
+def generate_live_round_stake(data, row_cts, c, r):
+    latest_round = int(data['roundNumber'].max())
+    select_round = int(latest_round - (2*r+c))
+    select_data = data[data['roundNumber']==select_round].reset_index(drop=True)
+    if len(select_data)>0:
+        payout_sum = select_data['payout'].sum().round(3)
+        stake_sum = select_data['stake'].sum().round(3)
+        if payout_sum >= 0:
+            payout_color = 'green'
+        else:
+            payout_color = 'red'
+
+        space = ' '*5
+        content_str = f'#### Round: {select_round}{space}Stake: {stake_sum}{space}Payout: <span style="color:{payout_color}">{payout_sum}</span> NMR'
+        row_cts[c].markdown(content_str, unsafe_allow_html=True)
+        select_data = select_data.drop(['roundNumber'], axis=1).sort_values(by='payout', ascending=False)
+        row_cts[c].dataframe(select_data, height=max_table_height-100)
+
+
+
+def round_view(data, select_perview, select_metric=None):
+    num_cols = 2
+    num_rows = 2
+    for r in range(num_rows):
+        row_cts = st.columns(num_cols)
+        for c in range(num_cols):
+            if select_perview=='round_result':
+                generate_round_table(data, row_cts, c, r)
+            if select_perview=='dailyscore_metric':
+                generate_dailyscore_metrics(data, row_cts, c, r)
+            if select_perview=='metric_view':
+                generate_round_metrics(data, row_cts, c, r)
+            if select_perview=='dailyscore_chart':
+                dailyscore_chart(data, row_cts, c, r, select_metric)
+            if select_perview=='live_round_stake':
+                 generate_live_round_stake(data, row_cts, c, r)
+
+
+def score_overview():
+    if 'model_data' in st.session_state:
+        data = st.session_state['model_data'].copy()
+        data = data.drop_duplicates(['model', 'roundNumber'], keep='first')
+        roundview =  st.expander('round performance overview', expanded=True)
+        with roundview:
+            round_view(data, 'round_result')
+    else:
+        st.write('model data missing, please go to the Dowanload Score Data section to download model data first')
+
+def metric_overview():
+    if 'model_data' in st.session_state:
+        data = st.session_state['model_data'].copy()
+        st.subheader('Select Round Data')
+        latest_round = int(data['roundNumber'].max())
+        earliest_round = int(data['roundNumber'].min())
+        if (latest_round - earliest_round) > 10:
+            # suggest_round = int(latest_round - (latest_round - earliest_round) / 2)
+            suggest_round = 280
+        else:
+            suggest_round = earliest_round
+        select_rounds = st.slider('select a round', earliest_round, latest_round, (suggest_round, latest_round - 1), 1)
+        data=data.drop_duplicates(['model', 'roundNumber'], keep='first')
+        data = data[(data['roundNumber'] >= select_rounds[0]) & (data['roundNumber'] <= select_rounds[1])].reset_index(drop=True)
+        roundmetrics_data = get_roundmetric_data(data)
+        min_count = int(roundmetrics_data['count'].min())
+        max_count = int(roundmetrics_data['count'].max())
+        if min_count < max_count:
+            select_minround = st.sidebar.slider('miminum number of rounds', min_count, max_count, min_count, 1)
+        else:
+            select_minround = min_count
+        roundmetrics_data = roundmetrics_data[roundmetrics_data['count'] >= select_minround].reset_index(drop=True)
+        metricview_exp = st.expander('metric overview', expanded=True)
+        dataview_exp = st.expander('full data view', expanded=False)
+        with metricview_exp:
+            round_view(roundmetrics_data, 'metric_view')
+        with dataview_exp:
+            st.write(roundmetrics_data)
+    else:
+        st.write('model data missing, please go to the Dowanload Score Data section to download model data first')
+
+
+def data_operation():
+    # top_lb, top_tp3m, top_tp1y, special_list = sidebar_data_picker()
+    full_model_list = st.session_state['models']
+    latest_round = project_utils.latest_round
+    models = []
+    benchmark_opt = st.sidebar.checkbox('download default models', value=True)
+    if benchmark_opt:
+        model_dict = default_model_picker()
+        for k in model_dict.keys():
+            models += model_dict[k]
+    models = models + model_fast_picker(full_model_list)
+    if len(models)>0:
+        model_selection = st.multiselect('select models', st.session_state['models'], default=models)
+    suggest_min_round = 182 #latest_round-50
+    min_round, max_round = st.slider('select tournament rounds', 200, latest_round, (suggest_min_round, latest_round), 1)
+    roundlist = [i for i in range(max_round, min_round-1, -1)]
+    download = st.button('download data of selected models')
+    st.sidebar.subheader('configuration')
+    show_info=st.sidebar.checkbox('show background data', value=False)
+    # update_numeraiti_data = st.sidebar.checkbox('update numerati data', value=True)
+    # update_model_data = st.sidebar.checkbox('update model data', value=True)
+    # update_model_data =
+
+    model_df = get_saved_data()
+    if download and len(model_selection)>0:
+        # if update_model_data:
+        with st.spinner('downloading model round results'):
+            model_df = []
+            model_df = download_model_round_result(model_selection, roundlist, show_info)
+
+    prjreload = st.sidebar.button('reload config')
+    if prjreload:
+        project_utils.reload_project()
+    if len(model_df)>0:
+        rename_dict = {'corrPercentile': 'corr_pct', 'correlation':'corr', 'corrWMetamodel':'corr_meta', 'mmcPercentile':'mmc_pct', 'tcPercentile':'tc_pct', 'fncV3Percentile':'fncV3_pct'}
+        model_df.rename(columns=rename_dict, inplace=True)
+        model_df['corrmmc'] = model_df['corr'] + model_df['mmc']
+        model_df['corrmmc_avg_pct'] = (model_df['corr_pct'] + model_df['mmc_pct'])/2
+        model_df['corrtc'] = model_df['corr'] + model_df['tc']
+        model_df['corrtc_avg_pct'] = (model_df['corr_pct'] + model_df['tc_pct'])/2
+        # st.write(model_df.head(5))
+        # ord_cols = ['model','corr', 'mmc', 'tc', 'corrmmc', 'corrtc', 'corr_pct', 'tc_pct',  'corrtc_avg_pct','corr_meta', 'mmc_pct', 'corrmmc_avg_pct', 'roundNumber', 'roundResolveTime']
+        ord_cols = ['model','corr', 'tc',  'corrtc', 'corr_pct', 'tc_pct',  'corrtc_avg_pct','corr_meta',  'fncV3', 'fncV3_pct','corrmmc_avg_pct', 'roundNumber', 'roundResolveTime', 'mmc', 'corrmmc','mmc_pct']
+
+        model_df = model_df[ord_cols]
+        if project_config.SAVE_LOCAL_COPY:
+            try:
+                project_utils.pickle_data(project_config.MODEL_ROUND_RESULT_FILE, model_df)
+            except:
+                pass
+        st.session_state['model_data'] = model_df
+
+    if show_info:
+        st.text('list of models being tracked')
+        st.write(model_dict)
+        try:
+            dshape = st.session_state['model_data'].shape
+            st.write(f'downloaded model result data shape is {dshape}')
+            st.write(model_df)
+        except:
+            st.write('model data was not retrieved')
+
+    if len(model_df)>0:
+        get_performance_data_status(model_df)
+    return None
+
+def get_saved_data():
+    res = []
+    if os.path.isfile(project_config.MODEL_ROUND_RESULT_FILE):
+        res = project_utils.load_data(project_config.MODEL_ROUND_RESULT_FILE)
+        st.session_state['model_data'] = res
+    return res
+
+def get_performance_data_status(df):
+    st.sidebar.subheader('model data summary')
+    # latest_date = df['date'][0].strftime(project_config.DATETIME_FORMAT3)
+    model_num = df['model'].nunique()
+    round_num = df['roundNumber'].nunique()
+    latest_round = df['roundNumber'].max()
+    # st.sidebar.text(f'latest date: {latest_date}')
+    st.sidebar.text(f'number of models: {model_num}')
+    st.sidebar.text(f'number of rounds: {round_num}')
+    st.sidebar.text(f'latest round: {latest_round}')
+    return None
+
+
+def download_model_round_result(models, roundlist, show_info):
+    model_df = []
+    model_dfs = []
+    my_bar = st.progress(0.0)
+    my_bar.progress(0.0)
+    percent_complete = 0.0
+    for i in range(len(models)):
+        message = ''
+        try:
+            model_res = numerapi_utils.daily_submissions_performances_V3(models[i])
+            if len(model_res) > 0:
+                cols = ['model'] + list(model_res[0].keys())
+                model_df = pd.DataFrame(model_res)
+                model_df['model'] = models[i]
+                model_df = model_df[cols]
+                model_dfs.append(model_df)
+            else:
+                message = f'no result found for model {models[i]}'
+        except Exception:
+            # if show_info:
+            #     st.write(f'error while getting result for {models[i]}')
+            except_msg = traceback.format_exc()
+            message = f'error while getting result for {models[i]}: {except_msg}'
+        if show_info and len(message) > 0:
+            st.info(message)
+        percent_complete += 1 / len(models)
+        if i == len(models) - 1:
+            percent_complete = 1.0
+        time.sleep(0.1)
+        my_bar.progress(percent_complete)
+        model_df = pd.concat(model_dfs, axis=0).sort_values(by=['roundNumber'], ascending=False).reset_index(drop=True)
+        model_df['roundResolveTime'] = pd.to_datetime(model_df['roundResolveTime'])
+        model_df['roundResolveTime'] = model_df['roundResolveTime'].dt.strftime(project_config.DATETIME_FORMAT3)
+        model_df = model_df[model_df['roundNumber'].isin(roundlist)].reset_index(drop=True)
+    return model_df
+
+def chart_pxline(data, x, y, color, hover_data=None, x_range=None):
+    fig = px.line(data, x=x, y=y, color=color, hover_data=hover_data)
+    fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='white', height = max_height, margin=dict(l=0, r=10, t=20, b=20))
+    fig.update_xaxes(showgrid=False, range=x_range)
+    fig.update_yaxes(gridcolor='grey')
+    return fig
+
+
+def roundresult_chart(data, model_selection):
+
+    round_data = data[data['model'].isin(model_selection)].drop_duplicates(['model', 'roundNumber'],                                                                           keep='first').reset_index(drop=True)
+    min_round = int(round_data['roundNumber'].min())
+    max_round = int(round_data['roundNumber'].max())
+    suggest_min_round = max_round - 20
+    if min_round == max_round:
+        min_round = max_round - 20
+
+    min_selectround, max_selectround = st.slider('select plotting round range', min_round, max_round,
+                                                 (suggest_min_round, max_round), 1)
+
+    select_metric = st.selectbox('Choose a metric', list(histtrend_opt.keys()), index=0,
+                                 format_func=lambda x: histtrend_opt[x])
+    round_range = [min_selectround, max_selectround]
+    round_list = [r for r in range(min_selectround, max_selectround + 1)]
+    round_data = round_data[round_data['roundNumber'].isin(round_list)]
+    mean_df = round_data.groupby(['model'])[select_metric].agg('mean').reset_index()
+    mean_df[f'model avg.'] = mean_df['model'] + ': ' + mean_df[select_metric].round(5).astype(str)
+    mean_df['mean'] = mean_df[select_metric]
+    merge_cols = ['model', 'model avg.', 'mean']
+    round_data = round_data.merge(right=mean_df[merge_cols], on='model', how='left').sort_values(by=['mean','model', 'roundNumber'], ascending=False)
+    fig = chart_pxline(round_data, 'roundNumber', y=select_metric, color='model avg.', hover_data=list(histtrend_opt.keys())+['roundResolveTime'],x_range=round_range)
+    if fig is not None:
+        st.plotly_chart(fig, use_container_width=True)
+
+
+
+
+
+
+def histtrend():
+    # default_models = ['yxbot']
+    # models = default_models.copy()
+    data = st.session_state['model_data'].copy()
+    models = data['model'].unique().tolist()
+    model_selection = []
+    default_models = model_fast_picker(models)
+    if len(models)>0:
+        if len(default_models)==0:
+            default_models = [models[0]]
+        model_selection = st.sidebar.multiselect('select models for chart', models, default=default_models)
+
+    if len(model_selection)>0:
+        roundresult_chart(data, model_selection)
+
+        # fig = px.line(df, x='roundNumber', y='corr', color='model', hover_data=['corr_pct'])
+        # st.write(model_selection)
+    else:
+        if len(model_selection)==0:
+            st.info('please select some models from the dropdown list')
+        else:
+            st.info('model result data file missing, or no model is selected')
+
+    # st.write(models)
+
+
+
+def model_evaluation():
+    data = st.session_state['model_data'].copy()
+    models = data['model'].unique().tolist()
+    model_selection = []
+    default_models = model_fast_picker(models)
+    mean_scale = [-0.05, 0.1]
+    count_scale = [1, 50]
+    sharpe_scale = [-0.2, 2]
+    pct_scale = [0, 1]
+    radar_scale = [0, 5]
+
+    if len(models)>0:
+        if len(default_models)==0:
+            default_models = [models[0]]
+        model_selection = st.sidebar.multiselect('select models for chart', models, default=default_models)
+
+    if len(model_selection)>0:
+        round_data = data[data['model'].isin(model_selection)].drop_duplicates(['model', 'roundNumber'],keep='first').reset_index(drop=True)
+        min_round = int(round_data['roundNumber'].min())
+        max_round = int(round_data['roundNumber'].max())
+        suggest_min_round = max_round - 20
+        if min_round == max_round:
+            min_round = max_round - 20
+
+        min_selectround, max_selectround = st.slider('select plotting round range', min_round, max_round,
+                                                     (suggest_min_round, max_round), 1)
+        round_list = [r for r in range(min_selectround, max_selectround+1)]
+        # defaultlist = ['corr_sharpe', 'tc_sharpe',  'corrtc_sharpe','corr_mean', 'tc_mean' 'corrtc_mean', 'corrtc_avg_pct','count']
+
+        defaultlist = ['corr_sharpe', 'tc_sharpe',  'corrtc_sharpe', 'corr_mean', 'tc_mean', 'corrtc_mean', 'corrtc_avg_pct_mean']
+
+        select_metrics = st.multiselect('Metric Selection', list(model_eval_opt.keys()),
+                                     format_func=lambda x: model_eval_opt[x], default=defaultlist)
+
+
+        round_data = round_data[round_data['roundNumber'].isin(round_list)].reset_index(drop=True)
+        #'need normalised radar chart + tabular view here
+        roundmetric_df = get_roundmetric_data(round_data).sort_values(by='corrtc_sharpe', ascending=False).reset_index(drop=True)
+
+        radarmetric_df = roundmetric_df.copy(deep=True)
+        for col in select_metrics:
+            if 'mean' in col:
+                use_scale = mean_scale
+            if 'sharpe' in col:
+                use_scale = sharpe_scale
+            if 'pct' in col:
+                use_scale = pct_scale
+            if 'count' in col:
+                use_scale = count_scale
+            radarmetric_df[col] = radarmetric_df[col].apply(lambda x: project_utils.rescale(x, use_scale, radar_scale))
+        select_metrics_name = [model_eval_opt[i] for i in select_metrics]
+        radarmetric_df.rename(columns=model_eval_opt, inplace=True)
+        roundmetric_df.rename(columns=model_eval_opt, inplace=True)
+
+        fig = go.Figure()
+        for i in range(len(radarmetric_df)):
+            fig.add_trace(go.Scatterpolar(
+                r=radarmetric_df.loc[i, select_metrics_name].values,
+                theta=select_metrics_name,
+                fill='toself',
+                name=radarmetric_df['model'].values[i]
+            ))
+
+        fig.update_polars(
+            radialaxis=dict(visible=True, autorange=False, #type='linear',
+                            range=[0,5])
+        )
+
+        fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='aliceblue',
+                          height=max_height+100,
+                          margin=dict(l=0, r=10, t=20, b=20), showlegend=True)
+
+        st.plotly_chart(fig, use_container_width=True)
+        st.text('Calculated Metrics')
+        st.dataframe(roundmetric_df[['model'] + select_metrics_name], height=max_table_height)
+        st.text('Rescaled Metrics on Chart')
+        st.dataframe(radarmetric_df[['model'] + select_metrics_name], height=max_table_height)
+
+        # st.write(select_metrics)
+
+
+def get_portfolio_overview(models, onlylatest=True):
+    res_df = []
+    my_bar = st.progress(0.0)
+    my_bar.progress(0.0)
+    percent_complete = 0.0
+    for i in range(len(models)):
+        m = models[i]
+        try:
+            if onlylatest:
+                # mdf = numerapi_utils.get_model_history(m).loc[0:0]
+                mdf = numerapi_utils.get_model_history_v3(m).loc[0:0]
+            else:
+                # mdf = numerapi_utils.get_model_history(m)
+                mdf = numerapi_utils.get_model_history_v3(m)
+            res_df.append(mdf)
+        except:
+            # st.info(f'no information for model {m} is available')
+            pass
+        percent_complete += 1 / len(models)
+        if i == len(models) - 1:
+            percent_complete = 1.0
+        time.sleep(0.1)
+        my_bar.progress(percent_complete)
+    try:
+        res_df = pd.concat(res_df, axis=0)
+        res_df['profitability'] = res_df['realised_pl']/(res_df['current_stake']-res_df['realised_pl'])
+        cols = ['model', 'date', 'current_stake', 'floating_stake', 'floating_pl', 'realised_pl', 'profitability', 'roundNumber', 'roundResolved', 'payout']
+
+        # res_df['date'] = res_df['date'].dt.date
+        if onlylatest:
+            res_df = res_df.sort_values(by='floating_pl', ascending=False).reset_index(drop=True)
+            return res_df[cols]
+        else:
+            return res_df[cols]
+    except:
+        return []
+
+
+def get_stake_type(corr, mmc):
+    if mmc>0:
+        res = str(int(corr)) + 'xCORR ' + str(int(mmc)) +'xMMC'
+    else:
+        res = '1xCORR'
+    return res
+
+
+@st.cache(suppress_st_warning=True)
+def get_stake_by_liverounds(models):
+    latest_round_id = int(project_utils.get_latest_round_id())
+    roundlist = [i for i in range(latest_round_id, latest_round_id - 5, -1)]
+    res = []
+    my_bar = st.progress(0.0)
+    my_bar.progress(0.0)
+    percent_complete = 0.0
+    percent_part = 0
+    for r in roundlist:
+        for m in models:
+            percent_complete += 1 / (len(models)*len(roundlist))
+            try:
+                data = numerapi_utils.get_round_model_performance(r, m)
+                # print(f'successfuly extract for model {m} in round {r}')
+                res.append(data)
+            except:
+                pass
+                # print(f'no result found for model {m} in round {r}')
+            if percent_part == (len(models)*len(roundlist)) - 1:
+                percent_complete = 1.0
+            time.sleep(0.1)
+            my_bar.progress(percent_complete)
+            percent_part +=1
+    res_df = pd.DataFrame.from_dict(res).fillna(0)
+    res_df['payoutPending'] = res_df['payoutPending'].astype(np.float64)
+    res_df['selectedStakeValue'] = res_df['selectedStakeValue'].astype(np.float64)
+    res_df['stake_type'] = res_df.apply(lambda x: get_stake_type(x['corrMultiplier'], x['mmcMultiplier']),axis=1)
+    rename_dict = {'selectedStakeValue': 'stake', 'payoutPending': 'payout', 'correlation':'corr'}
+    res_df = res_df.rename(columns=rename_dict)
+    col_ord = ['model', 'roundNumber', 'stake', 'payout', 'stake_type',  'corr', 'mmc']
+    return res_df[col_ord]
+
+
+
+def get_stake_graph(data):
+    numfeats = ['current_stake', 'floating_stake', 'floating_pl', 'realised_pl']
+    stat1 = ['sum']
+    agg_rcp = [[['date'], numfeats, stat1]]
+
+    select_opt = st.selectbox('Select Time Span', list(stakeoverview_plot_opt.keys()), index=1, format_func=lambda x: stakeoverview_plot_opt[x])
+
+    res = project_utils.groupby_agg_execution(agg_rcp, data)['date']
+    w5delta = datetime.timedelta(weeks=5)
+    w13delta = datetime.timedelta(weeks=13)
+    date_w5delta = res['date'].max() - w5delta
+    date_w13delta = res['date'].max() - w13delta
+    y1delta = datetime.timedelta(weeks=52)
+    date_y1delta = res['date'].max() - y1delta
+
+    rename_dict = {'date_current_stake_sum': 'total_stake', 'date_floating_stake_sum': 'floating_stake',
+                   'date_floating_pl_sum': 'floating_pl', 'date_realised_pl_sum': 'realised_pl'}
+    res = res.rename(columns=rename_dict)
+    if select_opt == '1month':
+        res = res[res['date']>date_w5delta]
+    elif select_opt=='3month':
+        res = res[res['date']>date_w13delta]
+    elif select_opt=='1year':
+        res = res[res['date']>date_y1delta]
+    else:
+        pass
+
+    fig = make_subplots(specs=[[{"secondary_y": True}]])
+    fig.add_trace( go.Scatter(x=res['date'], y=res['floating_stake'], name="floating_stake"), secondary_y=False,)
+
+    fig.add_trace(go.Scatter(x=res['date'], y=res['total_stake'], name="total_stake"),secondary_y=False,)
+
+    fig.add_trace(go.Scatter(x=res['date'], y=res['realised_pl'], name="realised_pl"),secondary_y=True,)
+    fig.update_layout(plot_bgcolor='black', paper_bgcolor='black', font_color='white')
+    fig.update_xaxes(showgrid=False, range=None, nticks=30)
+    fig.update_yaxes(gridcolor='grey', title_text="total stake/floating stake/realised PL", secondary_y=False)
+    fig.update_yaxes(showgrid=False, title_text="realised PL", zeroline=False,secondary_y=True)
+    st.plotly_chart(fig, use_container_width=True)
+
+#
+# def live_round_stakeview(data):
+#     models = data
+#     latest_round_id = int(project_utils.get_latest_round_id())
+#     roundlist = [i for i in range(latest_round_id, latest_round_id-4, -1]
+
+
+def check_session_state(key):
+    # st.write(data)
+    if key in st.session_state:
+        return st.session_state[key]
+    else:
+        return None
+
+
+def stake_overview():
+    # data = st.session_state['models'].copy()
+    models = st.session_state['models'].copy()
+    model_selection = []
+    baseline_models = []
+    model_dict = default_model_picker()
+    for k in model_dict.keys():
+        baseline_models += model_dict[k]
+
+    default_models = model_fast_picker(models)
+
+    if len(models)>0:
+        # if len(default_models)==0:
+        #     default_models = baseline_models[0]
+        model_selection = st.sidebar.multiselect('select models for chart', models, default=default_models)
+
+    redownload_data = False
+    # download = st.sidebar.button('download stake data')
+    if len(model_selection) > 0:
+        if 'stake_df' not in st.session_state:
+            redownload_data = True
+        else:
+            if set(model_selection)!=st.session_state['stake_overview_models']:
+                redownload_data = True
+            else:
+                ovdf = st.session_state['stake_df']
+        if redownload_data:
+            ovdf = get_portfolio_overview(model_selection, onlylatest=False)
+            st.session_state['stake_df'] = ovdf
+            st.session_state['stake_overview_models'] = set(ovdf['model'].unique().tolist())
+
+        chartdf = ovdf.copy(deep=True)
+        ovdf = ovdf.drop_duplicates('model', keep='first')
+        ovdf = ovdf.sort_values(by='floating_pl', ascending=False).reset_index(drop=True)
+        if len(ovdf) > 0:
+            overview_cols = ['model', 'current_stake', 'floating_stake', 'floating_pl', 'realised_pl']
+            date_text = datetime.datetime.now().strftime(project_config.DATETIME_FORMAT3)
+            ovdf.drop(['date'], axis=1, inplace=True)
+            stake_cts = st.columns(2)
+            pl_cts = st.columns(2)
+            date_label = st.empty()
+            get_stake_graph(chartdf)
+            ovdf_exp = st.expander('stake data overview', expanded=True)
+            with ovdf_exp:
+                st.dataframe(ovdf[overview_cols], height=max_table_height)
+            total_current_stake = round(ovdf['current_stake'].sum(), 3)
+            total_floating_stake = round(ovdf['floating_stake'].sum(), 3)
+            rpl = round(ovdf['realised_pl'].sum(), 3)
+            fpl = round(ovdf['floating_pl'].sum(), 3)
+            current_stake_str = f'### Stake Balance: {total_current_stake:0.3f} NMR'
+            float_stake_str = f'### Floating Balance: {total_floating_stake:0.3f} NMR'
+            if rpl >= 0:
+                real_pl_color = 'green'
+            else:
+                real_pl_color = 'red'
+            if fpl >= 0:
+                float_pl_color = 'green'
+            else:
+                float_pl_color = 'red'
+            real_pl_str = f'### Realised P/L: <span style="color:{real_pl_color}">{rpl}</span> NMR'
+            float_pl_str = f'### Floating P/L: <span style="color:{float_pl_color}">{fpl}</span> NMR'
+            stake_cts[0].markdown(current_stake_str, unsafe_allow_html=True)
+            stake_cts[1].markdown(float_stake_str, unsafe_allow_html=True)
+            pl_cts[0].markdown(real_pl_str, unsafe_allow_html=True)
+            pl_cts[1].markdown(float_pl_str, unsafe_allow_html=True)
+            date_label.subheader(f'Date: {date_text}')
+        if st.sidebar.checkbox('show breakdown by live rounds', value=False):
+            liveround_exp = st.expander('show breakdown by live rounds (requires extra data downloading)',expanded=True)
+            with liveround_exp:
+                stake_models = ovdf['model'].tolist()
+                liveround_stake_df = get_stake_by_liverounds(stake_models)
+                round_view(liveround_stake_df,'live_round_stake')
+        if st.sidebar.checkbox('show resolved round summary', value=False):
+            resolvedround_exp = st.expander('show resolved rounds summary for selected model group', expanded=True)
+            with resolvedround_exp:
+                get_roundresolve_history(chartdf)
+                # st.write(chartdf)
+
+
+def get_roundresolve_history(data):
+    resolved_rounds = data[data['roundResolved'] == True]['roundNumber'].unique().tolist()
+    rsdf = data[data['roundResolved'] == True].reset_index(drop=True)
+    rs_date = rsdf[['date', 'roundNumber']].drop_duplicates('roundNumber').reset_index(drop=True)
+    numfeats = ['current_stake', 'payout']
+    stat1 = ['sum']
+    agg_rcp = [[['roundNumber'], numfeats, stat1]]
+    res = project_utils.groupby_agg_execution(agg_rcp, rsdf)['roundNumber'].sort_values(by='roundNumber',
+                                                                                        ascending=False)
+    res = res.merge(right=rs_date, on='roundNumber')
+
+    rename_dict = {'roundNumber': 'Round', 'roundNumber_current_stake_sum': 'Total Stake',
+                   'roundNumber_payout_sum': 'Round P/L', 'date': 'Resolved Date'}
+    res.rename(columns=rename_dict, inplace=True)
+    st.write(res)
+
+
+
+
+def app_setting():
+    pfm_exp = st.expander('Perormance Data Setting', expanded=True)
+    with pfm_exp:
+        pfm_default_model= st.checkbox('download data for default model', value=True)
+
+    stake_exp = st.expander('stake overview data setting', expanded=True)
+    if st.button('confirm settiong'):
+        st.session_state['pfm_default_model'] = pfm_default_model
+
+
+
+def performance_overview():
+    # st.sidebar.subheader('Choose a Table View')
+    select_app = st.sidebar.selectbox("", list(pfm_opt.keys()), index=0, format_func=lambda x: pfm_opt[x])
+    if select_app=='data_op':
+        data_operation()
+    if select_app=='liveround_view':
+        score_overview()
+    if select_app=='metric_view':
+        metric_overview()
+    if select_app=='historic_trend':
+        histtrend()
+    if select_app=='model_evaluation':
+        model_evaluation()
+
+
+
+def show_content():
+    st.sidebar.header('Dashboard Selection')
+    select_app = st.sidebar.selectbox("", list(app_opt.keys()), index=1, format_func=lambda x: app_opt[x])
+    if select_app=='performance_overview':
+        performance_overview()
+    if select_app=='stake_overview':
+        stake_overview()
+    if select_app=='app_setting':
+        app_setting()
+
+
+# main body
+# various configuration setting
+app_opt = {
+           'performance_overview' : 'Performance Overview',
+           'stake_overview': 'Stake Overview',
+           # 'app_setting':''
+           }
+
+
+pfm_opt = {
+    'data_op': 'Download Score Data',
+    'liveround_view': 'Round Overview',
+    'metric_view':'Metric Overview',
+    'historic_trend': 'Historic Trend',
+    'model_evaluation': 'Model Evaluation',
+}
+
+
+
+tbl_opt = {
+            'round_result':'Round Results',
+            'dailyscore_metric':'Daily Score Metrics',
+            'round_metric' : 'Round Metrics'
+}
+
+id_metric_opt = {
+                'id_corr_sharpe':'Daily Score corr sharpe',
+                'id_mmc_sharpe': 'Daily Score mmc sharpe',
+                'id_corrmmc_sharpe': 'Daily Score corrmmc sharpe',
+                'id_corr2mmc_sharpe': 'Daily Score corr2mmc sharpe',
+                'id_corrmmcpct_sharpe': 'Daily Score corrmmc avg pct sharpe',
+                'id_corr2mmcpct_sharpe': 'Daily Score corr2mmc avg pct sharpe',
+                'id_corrpct_sharpe':'Daily Score corr pct sharpe',
+                'id_mmcpct_sharpe': 'Daily Score mmc pct sharpe',
+}
+
+
+id_metric_score_dic = {
+                'id_corr_sharpe':'corr',
+                'id_mmc_sharpe': 'mmc',
+                'id_corrmmc_sharpe': 'corrmmc',
+                'id_corr2mmc_sharpe': 'corr2mmc',
+                'id_corrmmcpct_sharpe': 'cmavg_pct',
+                'id_corr2mmcpct_sharpe': 'c2mavg_pct',
+                'id_corrpct_sharpe':'corr_pct',
+                'id_mmcpct_sharpe': 'mmc_pct'
+}
+
+
+roundmetric_opt ={'corr':'Corr metrics',
+                  'tc': 'TC metrics',
+                  'corrtc': 'CorrTC metrics',
+                  'fncV3': 'FNCV3 metrics',
+                  'pct': 'Pecentage metrics',
+                  'corrmmc' : 'CorrMMC metrics',
+                  'mmc': 'MMC metrics'
+}
+
+
+histtrend_opt = {
+                'corr':'Correlation',
+                'mmc': 'MMC',
+                'tc' : 'TC',
+                'corr_pct': 'Correlation Percentile',
+                'tc_pct' : 'TC Percentile',
+                'mmc_pct':'MMC Percentile',
+                'corrmmc': 'Correlation+MMC',
+                'corrtc': 'Correlation+TC',
+                'corrtc_avg_pct': 'Correlation+TC Average Percentile',
+                'corrmmc_avg_pct': 'Correlation+MMC Average Percentile',
+
+}
+
+
+model_eval_opt = {
+        'corr_sharpe' : 'Correlation Sharpe',
+        'mmc_sharpe' : 'MMC Sharpe',
+        'tc_sharpe' : 'TC Sharpe',
+        'corrtc_sharpe': 'Correlation+TC Sharpe',
+        'corrmmc_sharpe' : 'Correlation+MMC Sharpe',
+        'corr_mean':'Avg. Correlation',
+        'tc_mean': 'Avg. TC',
+        'count': 'Number of Rounds',
+        'mmc_mean':'Avg. MMC',
+        'corrtc_mean': 'Avg. Correlation+TC',
+        'corrmmc_mean': 'Avg. Correlation+MMC',
+        'corr_pct_mean': 'Avg. Correlation Percentile',
+        'mmc_pct_mean': 'Avg. MMC Percentile',
+        'corrmmc_avg_pct_mean': 'Avg. Correlation+MMC Percentile',
+        'corrtc_avg_pct_mean': 'Avg. Correlation+TC Percentile',
+}
+
+stakeoverview_plot_opt = {
+    '1month':'1 Month',
+    '3month':'3 Months',
+    '1year':'1 Year',
+    'all':'Display all available data'
+}
+
+def show_session_status_info():
+    # 'raw_performance_data'
+    key1 = 'model_data'
+    key2 = 'models'
+    if check_session_state(key1) is None:
+        st.write(f'{key1} is None')
+    else:
+        st.write(f'{key1} shape is {st.session_state[key1].shape}')
+
+    if check_session_state(key2) is None:
+        st.write(f'{key2} is None')
+    else:
+        st.write(f'{key2} list has {len(st.session_state[key2])} models')
+    pass
+
+
+
+project_utils.reload_project()
+
+height_exp = st.sidebar.expander('Plots and tables setting', expanded=False)
+with height_exp:
+    max_height = st.slider('Please choose the height for plots', 100, 1000, 400, 50)
+    max_table_height = st.slider('Please choose the height for tables', 100, 1000, 500, 50)
+
+
+st.title('Numerai Dashboard')
+# key = 'pfm_default_model'
+# if check_session_state('pfm_default_model') is None:
+#     st.write('set value')
+#     st.session_state['pfm_default_model'] = True
+# else:
+#     st.write('use set value')
+#
+# st.write(st.session_state)
+
+df = get_saved_data()
+
+if check_session_state('models') is None:
+    with st.spinner('updating model list'):
+        st.session_state['models'] = numerapi_utils.get_lb_models()
+
+# debug purpose only
+# show_session_status_info()
+
+show_content()

project_tools/numerapi_utils.py

@@ -0,0 +1,414 @@
+import numerapi
+from numerapi import utils
+from project_tools import project_config, project_utils
+from typing import List, Dict
+import pandas as pd
+import numpy as np
+
+napi = numerapi.NumerAPI()
+
+
+# def get_round
+
+
+# depreciated
+# def get_model_history(model):
+#     res = napi.daily_user_performances(model)
+#     res = pd.DataFrame.from_dict(res)
+#     res['payoutPending'] = res['payoutPending'].astype(np.float64)
+#     res['payoutSettled'] = res['payoutSettled'].astype(np.float64)
+#     res['stakeValue'] = res['stakeValue'].astype(np.float64)
+#     res['deltaRatio'] = res['payoutPending'] / res['stakeValue']
+#     res['realised_pl'] = project_utils.series_reverse_cumsum(res['payoutSettled'])
+#     res['floating_pl'] = project_utils.series_reverse_cumsum(res['payoutPending']) - res['realised_pl']
+#     res['current_stake'] = res['stakeValue'] - res['floating_pl']
+#     rename_dict = {'stakeValue':'floating_stake'}
+#     res = res.rename(columns=rename_dict)
+#     # res['equity'] = res['stakeValue'] + res['floating_pl']
+#     # cols = res.columns.tolist()
+#     # res = res[['model'] + cols]
+#
+#     res['model'] = model
+#     cols = ['model', 'date', 'current_stake', 'floating_stake', 'payoutPending', 'floating_pl', 'realised_pl']
+#     res = res[cols]
+#     return res
+
+
+def get_portfolio_overview(models, onlylatest=True):
+    res_df = []
+    for m in models:
+        # try:
+        print(f'extracting information for model {m}')
+        if onlylatest:
+            mdf = get_model_history_v3(m).loc[0:0]
+        else:
+            mdf = get_model_history_v3(m)
+        res_df.append(mdf)
+        # except:
+        #     print(f'no information for model {m} is available')
+    if len(res_df)>0:
+        res_df = pd.concat(res_df, axis=0)
+        # res_df['date'] = res_df['date'].dt.date
+        if onlylatest:
+            return res_df.sort_values(by='floating_pl', ascending=False).reset_index(drop=True)
+        else:
+            return res_df.reset_index(drop=True)
+    else:
+        return None
+
+
+
+
+
+
+def get_competitions(tournament=8):
+    """Retrieves information about all competitions
+    Args:
+        tournament (int, optional): ID of the tournament, defaults to 8
+            -- DEPRECATED there is only one tournament nowadays
+    Returns:
+        list of dicts: list of rounds
+        Each round's dict contains the following items:
+            * datasetId (`str`)
+            * number (`int`)
+            * openTime (`datetime`)
+            * resolveTime (`datetime`)
+            * participants (`int`): number of participants
+            * prizePoolNmr (`decimal.Decimal`)
+            * prizePoolUsd (`decimal.Decimal`)
+            * resolvedGeneral (`bool`)
+            * resolvedStaking (`bool`)
+            * ruleset (`string`)
+    Example:
+        >>> NumerAPI().get_competitions()
+        [
+         {'datasetId': '59a70840ca11173c8b2906ac',
+          'number': 71,
+          'openTime': datetime.datetime(2017, 8, 31, 0, 0),
+          'resolveTime': datetime.datetime(2017, 9, 27, 21, 0),
+          'participants': 1287,
+          'prizePoolNmr': Decimal('0.00'),
+          'prizePoolUsd': Decimal('6000.00'),
+          'resolvedGeneral': True,
+          'resolvedStaking': True,
+          'ruleset': 'p_auction'
+         },
+          ..
+        ]
+    """
+    # self.logger.info("getting rounds...")
+
+    query = '''
+        query($tournament: Int!) {
+          rounds(tournament: $tournament) {
+            number
+            resolveTime
+            openTime
+            resolvedGeneral
+            resolvedStaking
+          }
+        }
+    '''
+    arguments = {'tournament': tournament}
+    result = napi.raw_query(query, arguments)
+    rounds = result['data']['rounds']
+    # convert datetime strings to datetime.datetime objects
+    for r in rounds:
+        utils.replace(r, "openTime", utils.parse_datetime_string)
+        utils.replace(r, "resolveTime", utils.parse_datetime_string)
+        utils.replace(r, "prizePoolNmr", utils.parse_float_string)
+        utils.replace(r, "prizePoolUsd", utils.parse_float_string)
+    return rounds
+
+
+def daily_submissions_performances(username: str) -> List[Dict]:
+    """Fetch daily performance of a user's submissions.
+    Args:
+        username (str)
+    Returns:
+        list of dicts: list of daily submission performance entries
+        For each entry in the list, there is a dict with the following
+        content:
+            * date (`datetime`)
+            * correlation (`float`)
+            * roundNumber (`int`)
+            * mmc (`float`): metamodel contribution
+            * fnc (`float`): feature neutral correlation
+            * correlationWithMetamodel (`float`)
+    Example:
+        >>> api = NumerAPI()
+        >>> api.daily_user_performances("uuazed")
+        [{'roundNumber': 181,
+          'correlation': -0.011765912,
+          'date': datetime.datetime(2019, 10, 16, 0, 0),
+          'mmc': 0.3,
+          'fnc': 0.1,
+          'correlationWithMetamodel': 0.87},
+          ...
+        ]
+    """
+    query = """
+              query($username: String!) {
+                v2UserProfile(username: $username) {
+                  dailySubmissionPerformances {
+                    date
+                    correlation
+                    corrPercentile
+                    roundNumber
+                    mmc
+                    mmcPercentile
+                    fnc
+                    fncPercentile
+                    correlationWithMetamodel
+                  }
+                }
+              }
+            """
+    arguments = {'username': username}
+    data = napi.raw_query(query, arguments)['data']['v2UserProfile']
+    performances = data['dailySubmissionPerformances']
+    # convert strings to python objects
+    for perf in performances:
+        utils.replace(perf, "date", utils.parse_datetime_string)
+    # remove useless items
+    performances = [p for p in performances
+                    if any([p['correlation'], p['fnc'], p['mmc']])]
+    return performances
+
+
+def daily_submissions_performances_V3(modelname: str) -> List[Dict]:
+    query = """
+              query($modelName: String!) {
+                v3UserProfile(modelName: $modelName) {
+                    roundModelPerformances{
+                        roundNumber
+                        roundResolveTime
+                        corr
+                        corrPercentile
+                        mmc
+                        mmcMultiplier
+                        mmcPercentile
+                        tc
+                        tcPercentile
+                        tcMultiplier
+                        fncV3
+                        fncV3Percentile
+                        corrWMetamodel
+                        payout
+                        roundResolved
+                        roundResolveTime
+                        corrMultiplier
+                        mmcMultiplier
+                        selectedStakeValue
+                    }
+                    stakeValue
+                    nmrStaked
+                }
+              }
+            """
+    arguments = {'modelName': modelname}
+    data = napi.raw_query(query, arguments)['data']['v3UserProfile']
+    performances = data['roundModelPerformances']
+    # convert strings to python objects
+    for perf in performances:
+        utils.replace(perf, "date", utils.parse_datetime_string)
+    # remove useless items
+    performances = [p for p in performances
+                    if any([p['corr'], p['tc'], p['mmc']])]
+    return performances
+
+
+def get_lb_models(limit=20000, offset=0):
+    query = """
+           query($limit: Int, $offset: Int){
+               v2Leaderboard(limit:$limit, offset:$offset){
+                   username
+               }           
+           }
+           """
+    arguments = {'limit':limit, 'offset':offset}
+    data = napi.raw_query(query, arguments)['data']['v2Leaderboard']
+    model_list = [i['username'] for i in data]
+    return model_list
+
+
+
+def get_round_model_performance(roundNumber: int, model: str):
+    query = """
+              query($roundNumber: Int!, $username: String!) {
+                  roundSubmissionPerformance(roundNumber: $roundNumber, username: $username) {
+                      corrMultiplier
+                      mmcMultiplier                      
+                      roundDailyPerformances{
+                          correlation
+                          mmc
+                          corrPercentile
+                          mmcPercentile
+                          payoutPending
+                       }
+                       selectedStakeValue
+                   }
+              }
+            """
+    arguments = {'roundNumber': roundNumber,'username': model}
+    data = napi.raw_query(query, arguments)['data']['roundSubmissionPerformance']
+    latest_performance = data['roundDailyPerformances'][-1] #[-1] ### issue with order
+    res = {}
+    res['model'] = model
+    res['roundNumber'] = roundNumber
+    res['corrMultiplier'] = data['corrMultiplier']
+    res['mmcMultiplier'] = data['mmcMultiplier']
+    res['selectedStakeValue'] = data['selectedStakeValue']
+    for key in latest_performance.keys():
+        res[key] = latest_performance[key]
+    return res
+
+
+
+
+def get_user_profile(username: str) -> List[Dict]:
+    """Fetch daily performance of a user's submissions.
+    Args:
+        username (str)
+    Returns:
+        list of dicts: list of daily submission performance entries
+        For each entry in the list, there is a dict with the following
+        content:
+            * date (`datetime`)
+            * correlation (`float`)
+            * roundNumber (`int`)
+            * mmc (`float`): metamodel contribution
+            * fnc (`float`): feature neutral correlation
+            * correlationWithMetamodel (`float`)
+    Example:
+        >>> api = NumerAPI()
+        >>> api.daily_user_performances("uuazed")
+        [{'roundNumber': 181,
+          'correlation': -0.011765912,
+          'date': datetime.datetime(2019, 10, 16, 0, 0),
+          'mmc': 0.3,
+          'fnc': 0.1,
+          'correlationWithMetamodel': 0.87},
+          ...
+        ]
+    """
+    query = """
+              query($username: String!) {
+                v2UserProfile(username: $username) {
+                  dailySubmissionPerformances {
+                    date
+                    correlation
+                    corrPercentile
+                    roundNumber
+                    mmc
+                    mmcPercentile
+                    fnc
+                    fncPercentile
+                    correlationWithMetamodel
+                  }
+                }
+              }
+            """
+    arguments = {'username': username}
+    data = napi.raw_query(query, arguments)['data']#['v2UserProfile']
+    # performances = data['dailySubmissionPerformances']
+    # # convert strings to python objects
+    # for perf in performances:
+    #     utils.replace(perf, "date", utils.parse_datetime_string)
+    # # remove useless items
+    # performances = [p for p in performances
+    #                 if any([p['correlation'], p['fnc'], p['mmc']])]
+    return data
+
+
+def download_dataset(filename: str, dest_path: str = None,
+                     round_num: int = None) -> None:
+    """ Download specified file for the current active round.
+
+    Args:
+        filename (str): file to be downloaded
+        dest_path (str, optional): complate path where the file should be
+            stored, defaults to the same name as the source file
+        round_num (int, optional): tournament round you are interested in.
+            defaults to the current round
+        tournament (int, optional): ID of the tournament, defaults to 8
+
+    Example:
+        >>> filenames = NumerAPI().list_datasets()
+        >>> NumerAPI().download_dataset(filenames[0]}")
+    """
+    if dest_path is None:
+        dest_path = filename
+
+    query = """
+    query ($filename: String!
+           $round: Int) {
+        dataset(filename: $filename
+                round: $round)
+    }
+    """
+    args = {'filename': filename, "round": round_num}
+
+    dataset_url = napi.raw_query(query, args)['data']['dataset']
+    utils.download_file(dataset_url, dest_path, show_progress_bars=True)
+    
+    
+    
+# function using V3UserProfile
+
+def model_payout_history(model):
+    napi = numerapi.NumerAPI()
+    query = """
+              query($model: String!) {
+                  v3UserProfile(modelName: $model) {
+                        roundModelPerformances{
+                            payout
+                            roundNumber
+                            roundResolved
+                            roundResolveTime
+                            corrMultiplier
+                            mmcMultiplier
+                            selectedStakeValue
+                        }
+                        stakeValue
+                        nmrStaked
+                   }
+              }
+            """
+    arguments = {'model': model}
+    payout_info = napi.raw_query(query, arguments)['data']['v3UserProfile']['roundModelPerformances']
+    payout_info = pd.DataFrame.from_dict(payout_info)
+    payout_info = payout_info[~pd.isnull(payout_info['payout'])].reset_index(drop=True)
+    return payout_info
+
+
+def get_model_history_v3(model):
+    res = model_payout_history(model)
+    res = pd.DataFrame.from_dict(res)
+    res['payout'] = res['payout'].astype(np.float64)
+    res['current_stake'] = res['selectedStakeValue'].astype(np.float64)
+    res['payout_cumsum'] = project_utils.series_reverse_cumsum(res['payout'])
+    res['date'] = pd.to_datetime(res['roundResolveTime']).dt.date
+
+    res['realised_pl'] = res['payout_cumsum']
+    latest_realised_pl = res[res['roundResolved'] == True]['payout_cumsum'].values[0]
+    res.loc[res['roundResolved'] == False, 'realised_pl'] = latest_realised_pl
+
+    res['floating_pl'] = 0
+    payoutPending_values = res[res['roundResolved'] == False]['payout'].values
+    payoutPending_cumsum = payoutPending_values[::-1].cumsum()[::-1]
+    res.loc[res['roundResolved'] == False, 'floating_pl'] = payoutPending_cumsum
+
+    res['model'] = model
+    #     res['floating_pl'] = res['current_stake'] + res['payoutPending']
+    res['floating_stake'] = res['current_stake'] + res['floating_pl']
+    cols = ['model', 'date', 'current_stake', 'floating_stake', 'payout', 'floating_pl', 'realised_pl', 'roundResolved',
+            'roundNumber']
+    res = res[cols]
+    return res
+
+
+
+
+
+

project_tools/project_config.py

@@ -0,0 +1,21 @@
+import os
+import sys
+sys.path.append(os.path.dirname(os.getcwd()))
+
+DATETIME_FORMAT1 = '%Y%m%d%H%M'
+DATETIME_FORMAT2 = '%Y/%m/%d %H:%M'
+DATETIME_FORMAT3 = '%Y-%m-%d'
+SAVE_LOCAL_COPY = True
+
+BENCHMARK_MODELS = ['integration_test', 'integration_test_7'] #'budbot_7'] #'integration_test_7'
+FEATURE_PATH = './feature_data/'
+MODEL_ROUND_RESULT_FILE = './feature_data/model_round_result.pkl'
+MODEL_DAILY_RESULT_FILE = './feature_data/model_daily_result.pkl'
+NUMERATI_FILE = './feature_data/numerati_data.pkl'
+
+NUMERATI_URL = 'https://raw.githubusercontent.com/woobe/numerati/master/data.csv'
+
+
+
+
+

project_tools/project_utils.py

@@ -0,0 +1,815 @@
+import numpy as np
+import pandas as pd
+import os
+import pickle
+import time
+from contextlib import contextmanager
+from importlib import reload
+import re
+from project_tools import project_config, project_utils, numerapi_utils
+import glob
+import matplotlib.pyplot as plt
+import seaborn as sns
+from random import randint, random
+import itertools
+import scipy
+from scipy.stats import ks_2samp
+from sklearn.metrics import log_loss, roc_auc_score, accuracy_score, mean_squared_error
+from sklearn.preprocessing import MinMaxScaler, StandardScaler
+from sklearn.pipeline import make_pipeline
+from sklearn import linear_model
+import datetime
+import json
+from collections import OrderedDict
+from os import listdir
+from os.path import isfile, join, isdir
+import glob
+import numerapi
+import itertools
+import io
+import requests
+from pathlib import Path
+from scipy.stats.mstats import gmean
+from typing import List, Dict
+
+
+napi = numerapi.NumerAPI() #verbosity="info")
+
+
+def get_time_string():
+    """
+    Generate a time string representation of the time of call of this function.
+    :param None
+    :return: a string that represent the time of the functional call.
+    """
+    now = datetime.datetime.now()
+    now = str(now.strftime('%Y%m%d%H%M'))
+    return now
+
+
+def reload_project():
+    """
+    utility function used during experimentation to reload various model when required, useful for quick experiment iteration
+    :return: None
+    """
+    reload(project_config)
+    reload(project_utils)
+    reload(numerapi_utils)
+
+@contextmanager
+def timer(name):
+    """
+    utility timer function to check how long a piece of code might take to run.
+    :param name: name of the code fragment to be timed
+    :yield: time taken for the code to run
+    """
+    t0 = time.time()
+    print('[%s] in progress' % name)
+    yield
+    print('[%s] done in %.6f s' %(name, time.time() - t0))
+
+
+
+def load_data(pickle_file):
+    """
+    load pickle data from file
+    :param pickle_file: path of pickle data
+    :return: data stored in pickle file
+    """
+    load_file = open(pickle_file, 'rb')
+    data = pickle.load(load_file)
+    return data
+
+
+def pickle_data(path, data, protocol=-1, timestamp=False, verbose=True):
+    """
+    Pickle data to specified file
+    :param path: full path of file where data will be pickled to
+    :param data: data to be pickled
+    :param protocol: pickle protocol, -1 indicate to use the latest protocol
+    :return: None
+    """
+    file = path
+    if timestamp:
+        base_file = os.path.splitext(file)[0]
+        time_str = '_' + get_time_string()
+        ext = os.path.splitext(os.path.basename(file))[1]
+        file = base_file + time_str + ext
+
+    if verbose:
+        print('creating file %s' % file)
+
+    save_file = open(file, 'wb')
+    pickle.dump(data, save_file, protocol=protocol)
+    save_file.close()
+
+
+def save_json(path, data, timestamp=False, verbose=True, indent=2):
+    """
+    Save data to Json format
+    :param path: full path of file where data will be pickled to
+    :param data: data to be pickled
+    :param timestamp: if true, the timestamp will be saved as part of the file name
+    :param verbose: if true, print information about file creation
+    :param indent: specify the width of the indent in the resulted Json file
+    :return: None
+    """
+    file = path
+    if timestamp:
+        base_file = os.path.splitext(file)[0]
+        time_str = '_' + get_time_string()
+        ext = os.path.splitext(os.path.basename(file))[1]
+        file = base_file + time_str + ext
+    if verbose:
+        print('creating file %s' % file)
+    outfile = open(file, 'w')
+    json.dump(data, outfile, indent=indent)
+    outfile.close()
+
+
+def load_json(json_file):
+    """
+    load data from Json file
+    :param json_file: path of json file
+    :return: data stored in json file as python dictionary
+    """
+    load_file = open(json_file)
+    data = json.load(load_file)
+    load_file.close()
+    return data
+
+
+def create_folder(path):
+    Path(path).mkdir(parents=True, exist_ok=True)
+
+
+
+def glob_folder_filelist(path, file_type='', recursive=True):
+    """
+    utility function that walk through a given directory, and return list of files in the directory
+    :param path: the path of the directory
+    :param file_type: if not '', this function would only consider the file type specified by this parameter
+    :param recursive: if True, perform directory walk-fhrough recursively
+    :return absfile: a list containing absolute path of each file in the directory
+    :return base_files: a list containing base name of each file in the directory
+    """
+    if path[-1] != '/':
+        path = path +'/'
+    abs_files = []
+    base_files = []
+    patrn = '**' if recursive else '*'
+    glob_path = path + patrn
+    matches = glob.glob(glob_path, recursive=recursive)
+    for f in matches:
+        if os.path.isfile(f):
+            include = True
+            if len(file_type)>0:
+                ext = os.path.splitext(f)[1]
+                if ext[1:] != file_type:
+                    include = False
+            if include:
+                abs_files.append(f)
+                base_files.append(os.path.basename(f))
+    return abs_files, base_files
+
+
+def dir_compare(pathl, pathr):
+    files_pathl = set([f for f in listdir(pathl) if isfile(join(pathl, f))])
+    files_pathr = set([f for f in listdir(pathr) if isfile(join(pathr, f))])
+    return list(files_pathl-files_pathr), list(files_pathr-files_pathl)
+
+
+
+
+def lr_dir_sync(pathl, pathr):
+    files_lrddiff, files_rldiff = project_utils.dir_compare(pathl, pathr)
+    for f in files_lrddiff:
+        scr = pathl + f
+        dst = pathr + f
+        print('copying file %s' % scr)
+        copyfile(scr, dst)
+
+
+
+def copy_file_with_time(src_file, dst_file_name, des_path):
+    basename = os.path.splitext(os.path.basename(dst_file_name))[0]
+    ext_name = os.path.splitext(os.path.basename(dst_file_name))[1]
+    timestr = get_time_string()
+    des_name = '%s%s_%s%s' % (des_path, basename, timestr, ext_name)
+    # print(des_name)
+    copyfile(src_file, des_name)
+
+
+
+
+
+def find_filesfromfolder(target_dir, containtext):
+    absnames, basenames = glob_folder_filelist(target_dir)
+    result_filelist = []
+    for absname, basename in zip(absnames, basenames):
+        if containtext in basename:
+            result_filelist.append(absname)
+    # result_filelist = [f for f in total_filelist if containtext in f]
+    return result_filelist
+
+
+def cp_files_with_prefix(src_path, dst_path, prefix, ext):
+    abs_file_list, base_file_list = get_folder_filelist(src_path, file_type=ext)
+#     print(abs_file_list)
+    for src_file, base_file in zip(abs_file_list, base_file_list):
+        dst_file = dst_path + prefix + base_file
+        copyfile(src_file, dst_file)
+    return None
+
+
+
+def mv_files_with_prefix(src_path, dst_path, prefix, ext):
+    abs_file_list, base_file_list = get_folder_filelist(src_path, file_type=ext)
+#     print(abs_file_list)
+    for src_file, base_file in zip(abs_file_list, base_file_list):
+        dst_file = dst_path + prefix + base_file
+        move(src_file, dst_file)
+    return None
+
+
+
+def empty_folder(path):
+    if path[-1]!='*':
+        path = path + '*'
+    files = glob.glob(path)
+    for f in files:
+        os.remove(f)
+
+
+def rescale(n, range1, range2):
+    if n>range1[1]: #or n<range1[0]:
+        n=range1[1]
+    if n<range1[0]:
+        n=range1[0]
+    delta1 = range1[1] - range1[0]
+    delta2 = range2[1] - range2[0]
+    return (delta2 * (n - range1[0]) / delta1) + range2[0]
+
+
+
+def rmse(y_true, y_pred):
+    """
+    RMSE (Root Mean Square Error) evaluation function
+    :param y_true: label values
+    :param y_pred: prediction values
+    :return:  RMSE value of the input prediction values, evaluated against the input label values
+    """
+    return np.sqrt(mean_squared_error(y_true, y_pred))
+
+
+
+
+def str2date(date_str, dateformat='%Y-%m-%d'):
+    """
+    convert an input string in specified format into datetime format
+    :param date_str: the input string with certain specified format
+    :param dateformat: the format of the string which is used by the strptime function to do the type converson
+    :return dt_value: the datetime value that is corresponding to the input string and the specified format
+    """
+    dt_value = datetime.datetime.strptime(date_str, dateformat)
+    return dt_value
+
+
+def isnotebook():
+    """
+    Determine if the current python file is a jupyter notebook (.ipynb) or a python script (.py)
+    :return: return True if the the current python file is a jupyter notebook, otherwise return False
+    """
+    try:
+        shell = get_ipython().__class__.__name__
+        if shell == 'ZMQInteractiveShell':
+            return True   # Jupyter notebook
+        elif shell == 'TerminalInteractiveShell':
+            return False  # Terminal running IPython
+        else:
+            return False  # Other type (?)
+    except NameError:
+        return False
+
+
+
+def list_intersection(left, right):
+    """
+    take two list as input, conver them into sets, calculate the intersection of the two sets, and return this as a list
+    :param left: the first input list
+    :param right: the second input list
+    :return: the intersection set of elements for both input list, as a list
+    """
+    left_set = set(left)
+    right_set = set(right)
+    return list(left_set.intersection(right_set))
+
+
+def list_union(left, right):
+    """
+    take two list as input, conver them into sets, calculate the union of the two sets, and return this as a list
+    :param left: the first input list
+    :param right: the second input list
+    :return: the union set of elements for both input list, as a list
+    """
+    left_set = set(left)
+    right_set = set(right)
+    return list(left_set.union(right_set))
+
+
+def list_difference(left, right):
+    """
+    take two list as input, conver them into sets, calculate the difference of the first set to the second set, and return this as a list
+    :param left: the first input list
+    :param right: the second input list
+    :return: the result of difference set operation on elements for both input list, as a list
+    """
+    left_set = set(left)
+    right_set = set(right)
+    return list(left_set.difference(right_set))
+
+
+def is_listelements_identical(left, right):
+    equal_length = (len(left)==len(right))
+    zero_diff = (len(list_difference(left,right))==0)
+    return equal_length & zero_diff
+
+
+
+
+def np_corr(a, b):
+    """
+    take two numpy arrays, and compute their correlation
+    :param a: the first numpy array input
+    :param b: the second numpy array input
+    :return: the correlation between the two input arrays
+    """
+    return pd.Series(a).corr(pd.Series(b))
+
+
+
+def list_sort_values(a, ascending=True):
+    """
+    sort the value of a list in specified order
+    :param a: the input list
+    :param ascending: specified if the sorting is to be done in ascending or descending order
+    :return: the input list sorted in the specified order
+    """
+    return pd.Series(a).sort_values(ascending=ascending).tolist()
+
+
+def get_rank(data):
+    """
+    convert the values of a list or array into ranked percentage values
+    :param data: the input data in the form of a list or an array
+    :return: the return ranked percentage values in numpy array
+    """
+    ranks = pd.Series(data).rank(pct=True).values
+    return ranks
+
+
+
+def plot_feature_corr(df, features, figsize=(10,10), vmin=-1.0):
+    """
+    plot the pair-wise correlation matrix for specified features in a dataframe
+    :param df: the input dataframe
+    :param features: the list of features for which correlation matrix will be plotted
+    :param figsize: the size of the displayed figure
+    :param vmin: the minimum value of the correlation to be included in the plotting
+    :return: the pair-wise correlation values in the form of pandas dataframe, the figure will be plotted during the operation of this function.
+    """
+    val_corr = df[features].corr().fillna(0)
+    f, ax = plt.subplots(figsize=figsize)
+    sns.heatmap(val_corr, vmin=vmin, square=True)
+    return val_corr
+
+
+def decision_to_prob(data):
+    """
+    convert output value of a sklearn classifier (i.e. ridge classifier) decision function into probability
+    :param data: output value of decision function in the form of a numpy array
+    :return: value of probability in the form of a numpy array
+    """
+    prob = np.exp(data) / np.sum(np.exp(data))
+    return prob
+
+
+def np_describe(a):
+    """
+    provide overall statistic description of an input numpy value using the Describe method of Pandas Series
+    :param a: the input numpy array
+    :return: overall statistic description
+    """
+    return pd.Series(a.flatten()).describe()
+
+
+def ks_2samp_selection(train_df, test_df, pval=0.1):
+    """
+    use scipy ks_2samp function to select features that are statistically similar between the input train and test dataframe.
+    :param train_df: the input train dataframe
+    :param test_df: the input test dataframe
+    :param pval: the p value threshold use to decide which features to be selected. Only features with value higher than the specified p value will be selected
+    :return train_df: the return train dataframe with selected features
+    :return test_df: the return test dataframe with selected features
+    """
+    list_p_value = []
+    for i in train_df.columns.tolist():
+        list_p_value.append(ks_2samp(train_df[i], test_df[i])[1])
+    Se = pd.Series(list_p_value, index=train_df.columns.tolist()).sort_values()
+    list_discarded = list(Se[Se < pval].index)
+    train_df = train_df.drop(columns=list_discarded)
+    test_df = test_df.drop(columns=list_discarded)
+    return train_df, test_df
+
+
+
+def df_balance_sampling(df, class_feature, minor_class=1, sample_ratio=1):
+    """
+    :param df:
+    :param class_feature:
+    :param minor_class:
+    :param sample_ratio:
+    :return:
+    """
+    minor_df = df[df[class_feature] == minor_class]
+    major_df = df[df[class_feature] == (1 - minor_class)].sample(sample_ratio * len(minor_df))
+
+    res_df = minor_df.append(major_df)
+    res_df = res_df.sample(len(res_df)).reset_index(drop=True)
+    return res_df
+
+
+def prob2acc(label, probs, p=0.5):
+    """
+    calculate accuracy score  for probability predictions  with given threshold, as part of the process, the input probability predictions will be converted into discrete binary predictions
+    :param label: labels used to evaluate accuracy score
+    :param probs: probability predictions for which accuracy score will be calculated
+    :param p: the threshold to be used for convert probabilites into discrete binary values 0 and 1
+    :return acc: the computed accuracy score
+    :return preds: predictions in discrete binary value
+    """
+
+    preds = (probs >= p).astype(np.uint8)
+    acc = accuracy_score(label, preds)
+    return acc, preds
+
+
+
+def np_pearson(t,p):
+    vt = t - t.mean()
+    vp = p - p.mean()
+    top = np.sum(vt*vp)
+    bottom = np.sqrt(np.sum(vt**2)) * np.sqrt(np.sum(vp**2))
+    res = top/bottom
+    return res
+
+
+def df_get_features_with_str(df, ptrn):
+    """
+    extract list of feature names from a data frame that contain the specified regular expression pattern
+    :param df: the input dataframe of which features name to be analysed
+    :param ptrn: the specified regular expression pattern
+    :return: list of feature names that contained the specified regular expression
+    """
+    return [col for col in df.columns.tolist() if len(re.findall(ptrn, col)) > 0]
+
+
+def df_fillna_with_other(df, src_feature, dst_feature):
+    """
+    fill the NA values of a specified feature in a dataframe with values of another feature from the same row.
+    :param df: the input dataframe
+    :param src_feature: the specified feature of which NA value will be filled
+    :param dst_feature: the feature of which values will be used
+    :return: a dataframe with the specified feature's NA value being filled by values from the "dst_feature"
+    """
+    src_vals = df[src_feature].values
+    dst_vals = df[dst_feature].values
+    argwhere_nan = np.argwhere(np.isnan(dst_vals)).flatten()
+    dst_vals[argwhere_nan] = src_vals[argwhere_nan]
+    df[dst_feature] = dst_vals
+    return df
+
+
+
+def plot_prediction_prob(y_pred_prob):
+    """
+    plot probability prediction values using histrogram
+    :param y_pred_prob: the probability prediction values to be plotted
+    :return: None, the plot will be plotted during the operation of the function.
+    """
+    prob_series = pd.Series(data=y_pred_prob)
+    prob_series.name = 'prediction probability'
+    prob_series.plot(kind='hist', figsize=(15, 5), bins=50)
+    plt.show()
+    print(prob_series.describe())
+
+
+
+
+
+def df_traintest_split(df, split_var, seed=None, train_ratio=0.75):
+    """
+    perform train test split on a specified feature on a given dataframe wwith specified train ratio. Unique value of the specified feature will only present on either the resulted train or the test dataframe
+    :param df: the input dataframe to be split
+    :param split_var: the feature to be used as unique value to perform the split
+    :param seed: the random used to facilitate the train test split
+    :param train_ratio: the ratio of data to be split into the resulted train dataframe.
+    :return train_df: the resulted train dataframe after the split
+    :return test_df: the resulted test dataframe after the split
+    """
+    sv_list = df[split_var].unique().tolist()
+    train_length = int(len(sv_list) * train_ratio)
+    train_siv_list = pd.Series(df[split_var].unique()).sample(train_length, random_state=seed)
+    train_idx = df.loc[df[split_var].isin(train_siv_list)].index.values
+    test_idx = df.iloc[df.index.difference(train_idx)].index.values
+    train_df = df.loc[train_idx].copy().reset_index(drop=True)
+    test_df = df.loc[test_idx].copy().reset_index(drop=True)
+    return train_df, test_df
+
+
+
+# https://www.kaggle.com/gemartin/load-data-reduce-memory-usage
+def reduce_mem_usage(df, verbose=True, exceiptions=[]):
+    """ iterate through all the columns of a dataframe and modify the data type
+        to reduce memory usage.
+    """
+    np_input = False
+    if isinstance(df, np.ndarray):
+        np_input = True
+        df = pd.DataFrame(data=df)
+
+    start_mem = df.memory_usage().sum() / 1024 ** 2
+    col_id = 0
+    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
+    for col in df.columns:
+        if verbose: print('doing %d: %s' % (col_id, col))
+        col_type = df[col].dtype
+        try:
+            if (col_type != object) & (col not in exceiptions):
+                c_min = df[col].min()
+                c_max = df[col].max()
+                if str(col_type)[:3] == 'int':
+                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
+                        df[col] = df[col].astype(np.int8)
+                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
+                        df[col] = df[col].astype(np.int16)
+                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
+                        df[col] = df[col].astype(np.int32)
+                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
+                        df[col] = df[col].astype(np.int64)
+                else:
+                    if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
+                        #                         df[col] = df[col].astype(np.float16)
+                        #                     elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
+                        df[col] = df[col].astype(np.float32)
+                    else:
+                        df[col] = df[col].astype(np.float64)
+        #             else:
+        #                 df[col] = df[col].astype('category')
+        #                 pass
+        except:
+            pass
+        col_id += 1
+    end_mem = df.memory_usage().sum() / 1024 ** 2
+    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
+    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
+
+    if np_input:
+        return df.values
+    else:
+        return df
+
+
+
+def get_xgb_featimp(model):
+    imp_type = ['weight', 'gain', 'cover', 'total_gain', 'total_cover']
+    imp_dict = {}
+    try:
+        bst = model.get_booster()
+    except:
+        bst = model
+    feature_names = bst.feature_names
+    for impt in imp_type:
+        imp_dict[impt] = []
+        scores = bst.get_score(importance_type=impt)
+        for feature in feature_names:
+            if feature in scores.keys():
+                imp_dict[impt].append(scores[feature])
+            else:
+                imp_dict[impt].append(np.nan)
+    imp_df = pd.DataFrame(index=bst.feature_names, data=imp_dict)
+    return imp_df
+
+
+def get_df_rankavg(df):
+    idx = df.index
+    cols = df.columns.tolist()
+    rankavg_dict = {}
+    for col in cols:
+        rankavg_dict[col]=df[col].rank(pct=True).tolist()
+    rankavg_df = pd.DataFrame(index=idx, columns=cols, data=rankavg_dict)
+    rankavg_df['rankavg'] = rankavg_df.mean(axis=1)
+    return rankavg_df.sort_values(by='rankavg', ascending=False)
+
+
+def get_list_gmean(lists):
+    out = np.zeros((len(lists[0]), len(lists)))
+    for i in range(0, len(lists)):
+        out[:,i] = lists[i]
+    gmean_out = gmean(out, axis=1)
+    return gmean_out
+
+
+
+def generate_nwise_combination(items, n=2):
+    return list(itertools.combinations(items, n))
+
+
+def pairwise_feature_generation(df, feature_list, operator='addition', verbose=True):
+    feats_pair = generate_nwise_combination(feature_list, 2)
+    result_df = pd.DataFrame()
+    for pair in feats_pair:
+        if verbose:
+            print('generating %s of %s and %s' % (operator, pair[0], pair[1]))
+        if operator == 'addition':
+            feat_name = pair[0] + '_add_' + pair[1]
+            result_df[feat_name] = df[pair[0]] + df[pair[1]]
+        elif operator == 'multiplication':
+            feat_name = pair[0] + '_mulp_' + pair[1]
+            result_df[feat_name] = df[pair[0]] * df[pair[1]]
+        elif operator == 'division':
+            feat_name = pair[0] + '_div_' + pair[1]
+            result_df[feat_name] = df[pair[0]] / df[pair[1]]
+    return result_df
+
+
+def try_divide(x, y, val=0.0):
+    """
+    try to perform division between two number, and return a default value if division by zero is detected
+    :param x: the number to be used as dividend
+    :param y: the number to be used as divisor
+    :param val: the default output value
+    :return: the output value, the default value of val will be returned if division by zero is detected
+    """
+    if y != 0.0:
+        val = float(x) / y
+    return val
+
+
+def series_reverse_cumsum(a):
+    return a.fillna(0).values[::-1].cumsum()[::-1]
+
+
+def get_array_sharpe(values):
+    return values.mean()/values.std()
+
+
+#### NumerDash specific functions ###
+
+def calculate_rounddailysharpe_dashboard(df, lastround, earliest_round, score='corr'):
+    if score=='corr':
+        target = 'corr_sharpe'
+    elif score == 'corr_pct':
+        target = 'corr_pct_sharpe'
+    elif score=='mmc':
+        target = 'mmc_sharpe'
+    elif score=='mmc_pct':
+        target = 'mmc_pct_sharpe'
+    elif score=='corrmmc':
+        target = 'corrmmc_sharpe'
+    elif score=='corr2mmc':
+        target = 'corr2mmc_sharpe'
+    elif score=='cmavg_pct':
+        target = 'cmavgpct_sharpe'
+    elif score=='c2mavg_pct':
+        target = 'c2mavcpct_sharpe'
+
+    mean_feat = 'avg_sharpe'
+    sos_feat = 'sos'
+    df = df[(df['roundNumber'] >= earliest_round) & (df['roundNumber'] <= lastround)]
+    res = df.groupby(['model', 'roundNumber', 'group'])[score].apply(
+        lambda x: get_array_sharpe(x)).reset_index(drop=False)
+    res = res.rename(columns={score: target}).sort_values('roundNumber', ascending=False)
+    res = res.pivot(index=['model', 'group'], columns='roundNumber', values=target)
+    res.columns.name = ''
+    cols = [i for i in res.columns[::-1]]
+    res = res[cols]
+    res[mean_feat] = res[cols].mean(axis=1)
+    res[sos_feat] = res[cols].apply(lambda x: get_array_sharpe(x), axis=1)
+    res = res.drop_duplicates(keep='first').sort_values(by=sos_feat, ascending=False)
+    res.reset_index(drop=False, inplace=True)
+    return res[['model', 'group', sos_feat, mean_feat]+cols]
+
+
+
+def groupby_agg_execution(agg_recipies, df, verbose=True):
+    result_dfs = dict()
+    for groupby_cols, features, aggs in agg_recipies:
+        group_object = df.groupby(groupby_cols)
+        groupby_key = '_'.join(groupby_cols)
+        if groupby_key not in list(result_dfs.keys()):
+            result_dfs[groupby_key] = pd.DataFrame()
+        for feature in features:
+            rename_col = feature
+            for agg in aggs:
+                if isinstance(agg, dict):
+                    agg_name = list(agg.keys())[0]
+                    agg_func = agg[agg_name]
+                else:
+                    agg_name = agg
+                    agg_func = agg
+                if agg_name=='count':
+                    groupby_aggregate_name = '{}_{}'.format(groupby_key, agg_name)
+                else:
+                    groupby_aggregate_name = '{}_{}_{}'.format(groupby_key, feature, agg_name)
+                verbose and print(f'generating statistic {groupby_aggregate_name}')
+                groupby_res_df = group_object[feature].agg(agg_func).reset_index(drop=False)
+                groupby_res_df = groupby_res_df.rename(columns={rename_col: groupby_aggregate_name})
+                if len(result_dfs[groupby_key]) == 0:
+                    result_dfs[groupby_key] = groupby_res_df
+                else:
+                    result_dfs[groupby_key][groupby_aggregate_name] = groupby_res_df[groupby_aggregate_name]
+    return result_dfs
+
+
+def get_latest_round_id():
+    try:
+        all_competitions = numerapi_utils.get_competitions()
+        latest_comp_id = all_competitions[0]['number']
+    except:
+        print('calling numerai API unsuccessulf')
+        # local_data = load_data(project_config.DASHBOARD_MODEL_RESULT_FILE)
+        # latest_comp_id = local_data['roundNumber'].max()
+        latest_comp_id = 0
+    return int(latest_comp_id)
+
+#     except:
+
+latest_round = get_latest_round_id()
+
+
+
+
+def update_numerati_data(url=project_config.NUMERATI_URL, save_path=project_config.FEATURE_PATH):
+    content = requests.get(url).content
+    data = pd.read_csv(io.StringIO(content.decode('utf-8')))
+    save_file = os.path.join(save_path, 'numerati_data.pkl')
+    pickle_data(save_file, data)
+    return data
+
+
+
+
+def get_model_group(model_name):
+    cat_name = 'other'
+    if model_name in project_config.MODEL_NAMES+project_config.NEW_MODEL_NAMES:
+        cat_name = 'yx'
+    elif model_name in project_config.TOP_LB:
+        cat_name = 'top_corr'
+    elif model_name in project_config.IAAI_MODELS:
+        cat_name = 'iaai'
+    elif model_name in project_config.ARBITRAGE_MODELS:
+        cat_name = 'arbitrage'
+    elif model_name in project_config.MCV_MODELS:
+        cat_name = 'mcv'
+    # elif model_name in project_config.MM_MODELS:
+    #     cat_name = 'mm'
+    elif model_name in project_config.BENCHMARK_MODELS:
+        cat_name = 'benchmark'
+    elif model_name in project_config.TP3M:
+        cat_name = 'top_3m'
+    elif model_name in project_config.TP1Y:
+        cat_name = 'top_1y'
+    return cat_name
+
+
+def get_dashboard_data_status():
+    dashboard_data_tstr = 'NA'
+    nmtd_tstr = 'NA'
+    try:
+        dashboard_data_t = datetime.datetime.utcfromtimestamp(os.path.getctime(project_config.DASHBOARD_MODEL_RESULT_FILE))
+        dashboard_data_tstr = dashboard_data_t.strftime(project_config.DATETIME_FORMAT2)
+    except Exception as e:
+        print(e)
+        pass
+    try:
+        nmtd_t = datetime.datetime.utcfromtimestamp(os.path.getctime(project_config.NUMERATI_FILE))
+        nmtd_tstr = nmtd_t.strftime(project_config.DATETIME_FORMAT2)
+    except Exception as e:
+        print(e)
+        pass
+    return dashboard_data_tstr, nmtd_tstr
+
+
+
+
+
+
+
+
+
+
+
+
+
+

requirements.txt

@@ -0,0 +1,10 @@
+matplotlib==3.2.2
+numerapi==2.9.0
+numpy==1.20.0
+pandas==1.3.2
+Pillow==9.1.0
+plotly==4.14.3
+requests==2.25.1
+scikit_learn==1.0.2
+scipy==1.6.0
+seaborn==0.11.1