app.py

#!/usr/bin/env python
# coding: utf-8
import numpy as np
import io
import sys
import os
import pandas as pd
import datetime
import gc #garabage collector
from io import BytesIO
import panel as pn
import holoviews as hv
import hvplot.pandas
import xlsxwriter
from warnings import filterwarnings
'''
development env: panel serve script.py --autoreload
prod prep: panel convert script.py --to pyodide-worker --out pyodide
'''

filterwarnings("ignore")
# hv.extension('bokeh')
pn.extension( "plotly", template="fast")

pn.state.template.param.update(
    # site_url="",
    site="ModelMonitor",
    title="Classification Model Metrics",
    # favicon="https://raw.githubusercontent.com/firobeid/firobeid.github.io/main/docs/compose-plots/Resources/favicon.ico",
)
#######################
###UTILITY FUNCTIONS###
#######################
def percentage(df):
    def segment(df):
        return round(df["Count"]/df["Count"].sum(),4)
    df["percent"] = segment(df)
    return df

def AUC(group):
    from sklearn.metrics import roc_auc_score
    auc = roc_auc_score(group['TARGET'],group['SCORE'])
    # N = sum(group["N"])
    N = round(len(group.loc[group["TARGET"].notna()]),0)
    cols = ["AUC","Count"]
    # return trapezoidal_rule(FPR.to_numpy(),TPR.to_numpy())
    return pd.Series([auc, N], index = cols)

def ROC(group):
    from sklearn.metrics import roc_curve
    FPR,TPR,T = roc_curve(group['TARGET'],group['SCORE'])
    cols = ['TPR', 'FPR']
    return pd.concat([pd.Series(TPR),pd.Series(FPR)], keys = cols, axis = 1)

def ks(group):
    from scipy.stats import ks_2samp
    y_real = group['TARGET']
    y_proba = group['SCORE']
    
    df = pd.DataFrame()
    df['real'] = y_real
    df['proba'] = y_proba
    
    # Recover each class
    class0 = df[df['real'] == 0]
    class1 = df[df['real'] == 1]
    
    ks_ = ks_2samp(class0['proba'], class1['proba'])
    
    N = round(len(group.loc[group["TARGET"].notna()]),0)
    cols = ["KS","Count"]
    
    return pd.Series([ks_[0], N], index = cols)

def psi(df):
    '''
    https://mwburke.github.io/data%20science/2018/04/29/population-stability-index.html#:~:text=To%20calculate%20the%20PSI%20we,the%20percents%20in%20each%20bucket.
    '''
    df[df == 0] = 0.001
    sub = df.copy()
    sub = sub.iloc[:,:-1].sub(df.validation,axis = 0)
    div = df.copy()
    div= div.iloc[:,:-1].div(df.validation, axis=0)
    div = np.log(div)
    return (sub*div).sum(axis = 0)

def add_extremes_OOT(df, name:str, score:str):
    '''
    Mitigate bias in OOT/Serving/baseline set that might not have high confidence scores or low confidence scores
    :param: name: str, name of the appid column
    :param: score: str, name of the score column
    '''
    # df.loc[len(df.index)] = [np.nan, "Extreme_Case_Max", np.nan, np.nan, np.nan,994.0,0.0009,np.nan,np.nan,np.nan,np.nan]
    # df.loc[len(df.index)] = [np.nan, "Extreme_Case_Min", np.nan, np.nan, np.nan,158.0,0.9999,np.nan,np.nan,np.nan,np.nan]
    df.loc[len(df.index)] = [np.nan for i in range(0,df.shape[1])]
    df.loc[(len(df.index)-1), [name, score]] = ["Extreme_Case_Max", 0.0009]
    df.loc[len(df.index)] = [np.nan for i in range(0,df.shape[1])]
    df.loc[(len(df.index)-1), [name, score]] = ["Extreme_Case_Min", 0.9999]
    return df

# def last_3months(df):
#     from datetime import datetime
#     from dateutil.relativedelta import relativedelta
#     from pandas.tseries.offsets import MonthEnd

#     end_of_month = ((pd.Timestamp(datetime.now().strftime('%Y-%m-%d')) - pd.Timedelta(70, unit='D')) + relativedelta(months=-1)) + MonthEnd(0)
#     start_of_month = end_of_month + MonthEnd(-3) + relativedelta(days=1)
#     end_of_month = end_of_month +relativedelta(hours=23, minutes=59, seconds=59)
#     print('Start Month %r --- End Month %r' % (start_of_month, end_of_month))
#     try:
#         date_column = list(filter(lambda x:x.endswith("DATE"),gains_df.columns))[0]
#     except:
#         date_column = 'CREATED_DATE'
#     return df[df[date_column].between(start_of_month, end_of_month)]

def gains_table_proba(data=None,target=None, prob=None):
    data = data.copy()
    data['target0'] = 1 - data[target]
    data['bucket'] = pd.qcut(data[prob], 10)
    grouped = data.groupby('bucket', as_index = False)
    kstable = pd.DataFrame()
    kstable['min_prob'] = grouped.min()[prob]
    kstable['max_prob'] = grouped.max()[prob]
    kstable['count'] = grouped.count()['target0']
    kstable['cum_total']=(kstable['count'] / kstable['count'].sum()).cumsum()
    kstable['events']  = grouped.sum()[target]
    kstable['nonevents'] = grouped.sum()['target0']
    kstable['interval_rate'] = kstable['events'] / kstable['count']
    kstable = kstable.sort_values(by="min_prob", ascending=0).reset_index(drop = True)
    kstable['event_rate'] = (kstable.events / data[target].sum()).apply('{0:.2%}'.format)
    kstable['nonevent_rate'] = (kstable.nonevents / data['target0'].sum()).apply('{0:.2%}'.format)
    kstable['cum_eventrate']=(kstable.events / data[target].sum()).cumsum()
    kstable['cum_noneventrate']=(kstable.nonevents / data['target0'].sum()).cumsum()
    kstable['mid_point'] = np.nan
    kstable['KS'] = np.round(kstable['cum_eventrate']-kstable['cum_noneventrate'], 4) * 100

    #Formating
    kstable["cum_total"] = kstable["cum_total"].sort_values().values
    kstable = kstable.rename(columns={"min_prob":"low", "max_prob":"high"})
    kstable['mid_point'] = round((kstable['high'] + kstable['low']) / 2, 4)
    kstable['cum_eventrate']= kstable['cum_eventrate'].apply('{0:.2%}'.format)
    kstable['cum_noneventrate']= kstable['cum_noneventrate'].apply('{0:.2%}'.format)
    kstable.index = range(1,11)
    kstable.index.rename('Decile', inplace=True)
    pd.set_option('display.max_columns', 15)
    # print(kstable)
    #Display KS
    from colorama import Fore
    ks_3mnths = "KS is " + str(max(kstable['KS']))+"%"+ " at decile " + str((kstable.index[kstable['KS']==max(kstable['KS'])][0]))
    print("KS is " + str(max(kstable['KS']))+"%"+ " at decile " + str((kstable.index[kstable['KS']==max(kstable['KS'])][0])))
    kstable['cum_eventrate']= kstable['cum_eventrate'].str.replace("%","").astype(float)
    kstable['cum_noneventrate']= kstable['cum_noneventrate'].str.replace("%","").astype(float)
    kstable.index = list(range(10,0,-1))
    kstable = kstable.iloc[::-1]
    return(kstable, ks_3mnths)

def calculate_psi(expected, actual, buckettype='bins', buckets=10, axis=0):
    # https://www.kaggle.com/code/podsyp/population-stability-index
    '''Calculate the PSI (population stability index) across all variables
    Args:
       expected: numpy matrix of original values
       actual: numpy matrix of new values, same size as expected
       buckettype: type of strategy for creating buckets, bins splits into even splits, quantiles splits into quantile buckets
       buckets: number of quantiles to use in bucketing variables
       axis: axis by which variables are defined, 0 for vertical, 1 for horizontal
    Returns:
       psi_values: ndarray of psi values for each variable
    Author:
       Matthew Burke
       github.com/mwburke
       worksofchart.com
    '''

    def psi(expected_array, actual_array, buckets):
        '''Calculate the PSI for a single variable
        Args:
           expected_array: numpy array of original values
           actual_array: numpy array of new values, same size as expected
           buckets: number of percentile ranges to bucket the values into
        Returns:
           psi_value: calculated PSI value
        '''

        def scale_range (input, min, max):
            input += -(np.min(input))
            input /= np.max(input) / (max - min)
            input += min
            return input


        breakpoints = np.arange(0, buckets + 1) / (buckets) * 100

        if buckettype == 'bins':
            breakpoints = scale_range(breakpoints, np.min(expected_array), np.max(expected_array))
        elif buckettype == 'quantiles':
            breakpoints = np.stack([np.percentile(expected_array, b) for b in breakpoints])



        expected_percents = np.histogram(expected_array, breakpoints)[0] / len(expected_array)
        actual_percents = np.histogram(actual_array, breakpoints)[0] / len(actual_array)

        def sub_psi(e_perc, a_perc):
            '''Calculate the actual PSI value from comparing the values.
               Update the actual value to a very small number if equal to zero
            '''
            if a_perc == 0:
                a_perc = 0.0001
            if e_perc == 0:
                e_perc = 0.0001

            value = (e_perc - a_perc) * np.log(e_perc / a_perc)
            return(value)

        psi_value = np.sum(sub_psi(expected_percents[i], actual_percents[i]) for i in range(0, len(expected_percents)))

        return(psi_value)

    if len(expected.shape) == 1:
        psi_values = np.empty(len(expected.shape))
    else:
        psi_values = np.empty(expected.shape[axis])

    for i in range(0, len(psi_values)):
        if len(psi_values) == 1:
            psi_values = psi(expected, actual, buckets)
        elif axis == 0:
            psi_values[i] = psi(expected[:,i], actual[:,i], buckets)
        elif axis == 1:
            psi_values[i] = psi(expected[i,:], actual[i,:], buckets)

    return(psi_values)

    return round(10 **((158.313177 - UW5_Score) /274.360149), 18)

def lift_init(df:pd.DataFrame, baseline = None, is_baseline = True):
    from tqdm import tqdm
    # global standalone_scores_OOT
    cols = ['SCORE']
    
    lift_chart_data_OOT = pd.DataFrame()
    for q in tqdm([10,20,50,100]):
        # df_new["QUARTER"] = pd.PeriodIndex(df_new.CREATE_DATE, freq='Q')
        # fd = baseline.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, ), axis = 0).copy()
        # pd.cut(prod['SCORE'], bins = pd.qcut(baseline['SCORE'],10, retbins = True)[1]) 
        if is_baseline == True:
            # print(df)
            # print(df.dropna(subset = ['MONTHLY']))
            fd = df.dropna(subset = [period_metrics.value])[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]) , axis = 0).copy()
            fd = pd.concat([df.dropna(subset = [period_metrics.value])[period_metrics.value], df.dropna(subset = [period_metrics.value])['TARGET'], fd], axis = 1)
            fd = pd.concat([fd.groupby(x)['TARGET'].mean().fillna(0) for x in fd[cols]], axis = 1, keys = cols)
            fd.index.name = 'SCORE_BAND'
            
        else:
            # print(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel().shape)
            # print(pd.qcut(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel(),q=q, retbins = True))
            bins_ = pd.qcut(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel(),q=q, retbins = True)[1]
            fd = df.groupby([period_metrics.value]).apply(lambda col: col[cols].apply(lambda col: pd.cut(col, bins = bins_), axis = 0)).copy()
            # fd = df.groupby(period_metrics.value).apply(lambda col: col[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]), axis = 0)).copy()
            fd = pd.concat([df[period_metrics.value], df['TARGET'], fd], axis = 1)
            fd = fd.groupby(period_metrics.value).apply(lambda col: pd.concat([col.groupby(x)['TARGET'].mean().fillna(0) for x in col[cols]], axis = 1, keys = cols))
            fd.index.names = [period_metrics.value, 'SCORE_BAND']
        # fd['APPLICATION_MONTH'] = fd['APPLICATION_MONTH'].astype(str)
        fd = fd.reset_index()
        fd['BINS'] = q
        lift_chart_data_OOT = lift_chart_data_OOT.append(fd)
    if is_baseline == True:
        lift_chart_data_OOT[period_metrics.value] = 'Baseline'
       
    standalone_scores_OOT = lift_chart_data_OOT.melt(id_vars=[period_metrics.value,'BINS','SCORE_BAND'],value_vars=cols,
                                        var_name='SCORE', 
                                        value_name='BAD_RATE').dropna().reset_index(drop = True).copy()
    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(str)
    standalone_scores_OOT = pd.concat([standalone_scores_OOT["BINS"] + "-" + standalone_scores_OOT["SCORE_BAND"] + "-" + standalone_scores_OOT["SCORE"], 
                                       standalone_scores_OOT[[period_metrics.value,'BAD_RATE']]], axis = 1).rename(columns = {0:'BINS_SCOREBAND_SCORE'})
    standalone_scores_OOT = standalone_scores_OOT.pivot(index = 'BINS_SCOREBAND_SCORE', columns=period_metrics.value)['BAD_RATE'].reset_index()
    standalone_scores_OOT.index.name = ""
    standalone_scores_OOT.columns.name = ""
    standalone_scores_OOT = pd.concat([standalone_scores_OOT['BINS_SCOREBAND_SCORE'].str.split('-', expand=True),
                                   standalone_scores_OOT],axis = 1).rename(columns ={0:'BINS', 1: 'SCORE_BAND', 2: 'SCORE'}).drop(columns = 'BINS_SCOREBAND_SCORE')
    # standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']]#.astype(int)
    standalone_scores_OOT['BINS'] = standalone_scores_OOT['BINS']
    standalone_scores_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)
    return standalone_scores_OOT, lift_chart_data_OOT

def lift_init_plots(df:pd.DataFrame, is_baseline = True):
    from tqdm import tqdm
    # global standalone_scores_OOT
    cols = ['SCORE']

    lift_chart_data_OOT = pd.DataFrame()
    for q in tqdm([10,20,50,100]):
        # df_new["QUARTER"] = pd.PeriodIndex(df_new.CREATE_DATE, freq='Q')
        # fd = baseline.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, ), axis = 0).copy()
        # pd.cut(prod['SCORE'], bins = pd.qcut(baseline['SCORE'],10, retbins = True)[1]) 
        # fd = df.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]) , axis = 0).copy()
        if is_baseline == True:
            fd = df.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, labels=range(1, q + 1)), axis = 0).copy()
            fd = pd.concat([df.dropna(subset = period_metrics.value)[period_metrics.value], df.dropna(subset = period_metrics.value)['TARGET'], fd], axis = 1)
            fd = pd.concat([fd.groupby(x)['TARGET'].mean().fillna(0) for x in fd[cols]], axis = 1, keys = cols)
            fd.index.name = 'SCORE_BAND'
            
        else:
            fd = df.groupby(period_metrics.value).apply(lambda col: col[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, labels=range(1,q + 1)), axis = 0)).copy()
            fd = pd.concat([df[period_metrics.value], df['TARGET'], fd], axis = 1)
            fd = fd.groupby(period_metrics.value).apply(lambda col: pd.concat([col.groupby(x)['TARGET'].mean().fillna(0) for x in col[cols]], axis = 1, keys = cols))
            # print(fd.index)
            fd.index.names = [period_metrics.value, 'SCORE_BAND']
            # fd = fd.reset_index(names = ['APPLICATION_MONTH', 'SCORE_BAND'])  
        fd = fd.reset_index()
        # fd['APPLICATION_MONTH'] = fd['APPLICATION_MONTH'].astype(str)
        fd['BINS'] = q
        lift_chart_data_OOT = lift_chart_data_OOT.append(fd)
    if is_baseline == True:
        lift_chart_data_OOT[period_metrics.value] = 'Baseline'
    lift_chart_data_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)   
    standalone_scores_OOT = lift_chart_data_OOT.melt(id_vars=[period_metrics.value,'BINS','SCORE_BAND'],value_vars=cols,
                                        var_name='SCORE', 
                                        value_name='BAD_RATE').dropna().reset_index(drop = True).copy()
    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(str)
    standalone_scores_OOT = pd.concat([standalone_scores_OOT["BINS"] + "-" + standalone_scores_OOT["SCORE_BAND"] + "-" + standalone_scores_OOT["SCORE"], 
                                       standalone_scores_OOT[[period_metrics.value,'BAD_RATE']]], axis = 1).rename(columns = {0:'BINS_SCOREBAND_SCORE'})
    standalone_scores_OOT = standalone_scores_OOT.pivot(index = 'BINS_SCOREBAND_SCORE', columns=period_metrics.value)['BAD_RATE'].reset_index()
    standalone_scores_OOT.index.name = ""
    standalone_scores_OOT.columns.name = ""
    standalone_scores_OOT = pd.concat([standalone_scores_OOT['BINS_SCOREBAND_SCORE'].str.split('-', expand=True),
                                   standalone_scores_OOT],axis = 1).rename(columns ={0:'BINS', 1: 'SCORE_BAND', 2: 'SCORE'}).drop(columns = 'BINS_SCOREBAND_SCORE')
    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(int)
    standalone_scores_OOT['BINS'] = standalone_scores_OOT['BINS']
    standalone_scores_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)
    return standalone_scores_OOT

def save_csv(df, metric):
    from io import StringIO
    sio = StringIO()
    df.to_csv(sio)
    sio.seek(0)
    return pn.widgets.FileDownload(sio, embed=True, filename='%s.csv'%metric)

def get_xlsx(df1,df2,df3,df4,df5,df6):
    from io import BytesIO
    output = BytesIO()
    writer = pd.ExcelWriter(output, engine='xlsxwriter')
    df1.to_excel(writer, sheet_name="PSI")
    df2.to_excel(writer, sheet_name="AUC")
    df3.to_excel(writer, sheet_name="KS")
    df4.to_excel(writer, sheet_name="LABEL_DRIFT")
    df5.to_excel(writer, sheet_name="LABEL_Tables")
    df6.to_excel(writer, sheet_name="GAINS_Tables")
    writer.save() # Important!
    output.seek(0) # Important!
    return pn.widgets.FileDownload(output,embed=True, filename='results.csv', button_type="primary")


def expected_calibration_error(y, proba, bins = 'fd'):
  import numpy as np
  bin_count, bin_edges = np.histogram(proba, bins = bins)
  n_bins = len(bin_count)
  bin_edges[0] -= 1e-8 # because left edge is not included
  bin_id = np.digitize(proba, bin_edges, right = True) - 1
  bin_ysum = np.bincount(bin_id, weights = y, minlength = n_bins)
  bin_probasum = np.bincount(bin_id, weights = proba, minlength = n_bins)
  bin_ymean = np.divide(bin_ysum, bin_count, out = np.zeros(n_bins), where = bin_count > 0)
  bin_probamean = np.divide(bin_probasum, bin_count, out = np.zeros(n_bins), where = bin_count > 0)
  ece = np.abs((bin_probamean - bin_ymean) * bin_count).sum() / len(proba)
  return ece, bin_probamean, bin_ymean, bin_id, bin_count, bin_edges
###############################
###END OFF UTILITY FUNCTIONS###
###############################

text = """
#Classification Model Metrics
## AUTHOR: [`FIRAS ALI OBEID`](https://www.linkedin.com/in/feras-obeid/) 
###  GNU General Public License v3.0 (GPL-3.0)
#### Developed while working at [OppFi Inc.](https://www.oppfi.com/)

This tool performs ML model ,in production, monitoring across time, 
where production weeks/months/quarters are compared too a selective baseline.

1. Upload a CSV containing:

**(Date)** Highly Recommended but **optional** 
**(Score)** Probability Predictions  
**(Target)** Binary Target/True Label
 
2. Check the box if you CSV has a DATE column, otherwise dates are generated based on current timestamp and spanning back by 
timedelta of csv length in hourly frequency.

3. Choose & press the right columns in the `Select Boxes` below when you upload a csv

4. Select a baseline date slice **mandatory**. If your baseline is from a different time then the production time,
make sure to append it to the csv before uploading.

5. Press Get Metrics

6. Wait few seconds and analyze the updated charts
"""



# date = str(input('What is the name off the date column: ').upper())
# id_ = str(input('What is the name off the APP name/ID column: ').upper())
# score = str(input('What is the name off the score column (i.e UW5,DM_QL...): ').upper())
# target = str(input('What is the name off the Target column (i.e Real target values such as PD70_RATIO...: ').upper())

file_input = pn.widgets.FileInput(align='center')
date_selector = pn.widgets.Select(name='Select Date Column',)
check_date = pn.widgets.Checkbox(name = '<--',value = False) # T/F
target_selector = pn.widgets.Select(name='Select Target Variable(True Label)')
score_selector = pn.widgets.Select(name='Select Predictions Column(Raw Probaility)')
period_metrics = pn.widgets.Select(name='Select Period', options = ['MONTHLY','WEEKLY', 'QUARTERLY'])

date_range_ = pn.widgets.DateRangeSlider(name='Baseline Period',) #value=(start, end), start=start, end=end

random_seed = pn.widgets.IntSlider(name='Random Seed for Random Generated Data (OnSet)', value=42, start=0, end=1000, step=1)

button = pn.widgets.Button(name='Get Metrics')
widgets = pn.WidgetBox(
    pn.panel(text, margin=(0, 20)),
    pn.panel('**Check box if your data has a date column *before uploading the file* \n (otherwise keep it empty)**'),
    check_date,
    file_input,
    random_seed,
    pn.panel('\n'),
    date_selector,
    target_selector,
    score_selector,
    period_metrics,
    date_range_,
    button
)

# start, end = stocks.index.min(), stocks.index.max()
# year = pn.widgets.DateRangeSlider(name='Year', value=(start, end), start=start, end=end)
# ,id_:'ID',


def get_data():
    global df
    if file_input.value is None:
        np.random.seed(random_seed.value)
        try:
            df = pd.DataFrame({'DATE': pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = 9999)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H"),
                            'ID': [i for i in range(10000)],
                            'SCORE':np.random.uniform(size = 10000),
                            'TARGET': np.random.choice([0,1],10000, p=[0.9,0.1])})
        except:
            df = pd.DataFrame({'DATE': pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = 9999 + 1)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H"),
                            'ID': [i for i in range(10000)],
                            'SCORE':np.random.uniform(size = 10000),
                            'TARGET': np.random.choice([0,1],10000, p=[0.9,0.1])})            
        # df.to_csv("test_upload.csv")
    else:
        df = BytesIO()
        df.write(file_input.value)
        df.seek(0)
        try:
                df = pd.read_csv(df, error_bad_lines=False).apply(pd.to_numeric, errors='ignore')
        except:
                df = pd.read_csv(df, error_bad_lines=False)

        df = df.select_dtypes(exclude=["category"])
        df = df.replace([np.inf, -np.inf], np.nan)
        df.columns = [i.upper() for i in df.columns]
    return df

def update_target(event):
    df = get_data()
    cols = list(df.columns)
    date_selector.set_param(options=cols)
    target_selector.set_param(options=cols)
    score_selector.set_param(options=cols)
    # print(check_date.value)
    # print(type(df.DATE.min()))
    if check_date.value == True:
        date_column = [i.find("DATE") for i in df.columns] 
        date_column = [date_column.index(i) for i in [i for i in date_column if i !=-1]]
        if len(date_column) > 0:
            df = df.iloc[:,date_column].iloc[:,[0]]
            df.columns = ['DATE']
            print(type(df.DATE.min()))
            start, end = pd.Timestamp(df.DATE.min()),  pd.Timestamp(df.DATE.max())
            try:
                date_range_.set_param(value=(start, end), start=start, end=end)
            except:
                date_range_.set_param(value=(end, start), start=end, end=start)
        else:
            print('Creating synthetic dates')
            synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df))), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H") #remove len(df) - 1
            df['DATE'] = synthetic_date[:len(df)]
            start, end = df.DATE.min(), df.DATE.max()
            date_range_.set_param(value=(start, end), start=start, end=end)
    else:
        print('Creating synthetic dates')
        synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df))), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H")
        df['DATE'] = synthetic_date[:len(df)]
        start, end = df.DATE.min(), df.DATE.max()
        date_range_.set_param(value=(start, end), start=start, end=end)
file_input.param.watch(update_target, 'value')
update_target(None)

@pn.depends(button.param.clicks)
def run(_):
    print(random_seed.value)
    print(score_selector.value)
    df = get_data()
    try:
        if file_input.value is None:
            pass
        elif check_date.value == True:
            df = df.rename(columns={date_selector.value:'DATE',score_selector.value:'SCORE',target_selector.value:'TARGET'})
        else:
            synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df) - 1)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H")
            df['DATE'] = synthetic_date[:len(df)]
            df = df.rename(columns={score_selector.value:'SCORE',target_selector.value:'TARGET'})
    except Exception as e:
        return pn.pane.Markdown(f"""{e}""")
    try:
        df.DATE = pd.to_datetime(df.DATE, format="%Y-%m-%d %H:%M:%S", utc = True)
        # print(pd.to_datetime(df.DATE,utc = True))
        df["MONTHLY"] = df["DATE"].dt.strftime('%Y-%m')
        print(f"J - DAYS COUNT: {datetime.datetime.now() - pd.Timestamp('2023-03-06 03:27')}" )
        df['QUARTERLY'] = pd.PeriodIndex(df.DATE, freq='Q').astype(str)
        df['WEEKLY'] = pd.PeriodIndex(df.DATE, freq='W').astype(str)
    except Exception as e:
        return pn.pane.Markdown(f"""{e}""")    
    df = df.reset_index().rename(columns={df.index.name:'ID'}) #crate synthetic prediction ID for my code to run
    # df = df.dropna(subset = 'TARGET', axis = 1)
    df = df[~(df.TARGET.isna()) | (df.SCORE.isna())]
    if df.TARGET.nunique() > 2:
        df.TARGET = np.where(df.TARGET > 0 , 1 , 0)      
    df.SCORE = df.SCORE.astype(np.float64)

    

    # baselines
    # try:
    #     baseline = df.set_index('MONTHLY').loc[date_range_.value[0]: date_range_.value[1]].reset_index().copy()
    # except:
    #     baseline = df.copy()
    #     baseline = baseline.set_index('MONTHLY')
    #     baseline.index = pd.to_datetime(baseline.index)
    #     baseline = baseline.loc[date_range_.value[0]: date_range_.value[1]].reset_index()
    #     baseline["MONTHLY"] = baseline["MONTHLY"] .dt.strftime('%Y-%m')
    print(date_range_.value[0])
    print(date_range_.value[1])

    baseline = df.set_index('DATE').sort_index().loc[date_range_.value[0]: date_range_.value[1]].reset_index()
    print(baseline.DATE.min())
    print(baseline.DATE.max())
    print(df.DATE.max())

    # print(df.set_index('DATE').loc[date_range_.value[0]: date_range_.value[1]].index.max())
    #prods
    # prod = df.loc[~df.MONTHLY.isin(list(baseline.MONTHLY.unique()))].copy()
    prod_dates = df.set_index('DATE').sort_index().index.difference(baseline.set_index('DATE').index)
    # print(prod_dates)
    prod = df.set_index('DATE').loc[prod_dates].reset_index()
    if len(baseline) > len(prod):
        prod = baseline
    ##START##
    intiate = pn.pane.Alert('''### Baseline Period: \n%s to %s
    '''%(baseline.DATE.min(),baseline.DATE.max()), alert_type="info")
    intiate2 = pn.pane.Alert('''### Production Period: \n%s to %s
    '''%(prod.DATE.min(),prod.DATE.max()), alert_type="info")
    if prod.equals(baseline):
        intiate3 = pn.pane.Alert('''### Baseline Set is identical to Production Set \n Please choose a slice to be a baseline set''', alert_type="danger")
    else:
        intiate3 = None
    ##PSI##
    baseline_psi = baseline.copy()
    prod_psi = prod.copy()

    baseline_psi = add_extremes_OOT(baseline_psi, name = 'ID', score = 'SCORE')
    prod_psi["DEC_BANDS"] = pd.cut(prod_psi['SCORE'], bins = pd.qcut(baseline_psi['SCORE'],10, retbins = True)[1]) 
    prod_psi = prod_psi.groupby([period_metrics.value,
                                    "DEC_BANDS"]).agg(Count = ("DEC_BANDS",
                                    "count")).sort_index(level = 0).reset_index()
    prod_psi = prod_psi.groupby(period_metrics.value).apply(percentage).drop("Count",axis = 1)

    baseline_psi["DECILE"] = pd.cut(baseline_psi['SCORE'], bins = pd.qcut(baseline_psi['SCORE'],10, retbins = True)[1]) 
    baseline_psi = baseline_psi["DECILE"].value_counts()
    baseline_psi = baseline_psi / sum(baseline_psi)
    baseline_psi = baseline_psi.reset_index().rename(columns={"index":"DEC_BANDS", "DECILE": "percent"})
    baseline_psi[period_metrics.value] = "validation"
    baseline_psi = baseline_psi[[period_metrics.value, "DEC_BANDS", "percent"]]

    prod_psi = pd.concat([prod_psi,baseline_psi])

    prod_psi = prod_psi.pivot(index = "DEC_BANDS", columns=period_metrics.value)["percent"]
    if len(baseline) < len(prod):
        psi_ = psi(prod_psi).to_frame("%s_PSI"%period_metrics.value)
        psi_results = pn.widgets.DataFrame(psi_)
    else: 
        psi_ = pd.DataFrame()
        psi_results = pn.pane.Alert("### Choose a Baseline in the left banner to get PSI results", alert_type="warning")
    #CONFIGS
    baseline['QUARTERLY'] = 'Baseline: '+ baseline['QUARTERLY'].unique()[0] + '_' + baseline['QUARTERLY'].unique()[-1]
    baseline['MONTHLY'] = 'Baseline: '+ baseline['MONTHLY'].unique()[0] + '_' + baseline['MONTHLY'].unique()[-1]
    baseline['WEEKLY'] = 'Baseline: '+ baseline['WEEKLY'].unique()[0] + '_' + baseline['WEEKLY'].unique()[-1]
    #AUC
    auc_b = baseline.groupby([period_metrics.value]).apply(AUC)
    auc_p = prod.groupby([period_metrics.value]).apply(AUC)
    baseline_auc = pn.widgets.DataFrame(auc_b)
    prod_auc = pn.widgets.DataFrame(auc_p,name = 'AUC') #autosize_mode='fit_columns'
    
    from sklearn.metrics import roc_curve
    from holoviews import Slope
    b_label = baseline.MONTHLY.min()
    FPR,TPR,T = roc_curve(baseline['TARGET'],baseline['SCORE'])
    roc_baseline = pd.concat([pd.Series(TPR), pd.Series(FPR)], keys = ['TPR', 'FPR'], axis = 1)
    roc_baseline_p = roc_baseline.hvplot.line(x ='FPR', y = 'TPR', label = "Baseline", color = 'red')

    roc_plot = prod.groupby([period_metrics.value]).apply(ROC).reset_index(level = 0).hvplot.line(x ='FPR', y = 'TPR', title = "%s ROC (Production VS %s)"%(period_metrics.value, b_label),
                                                        groupby = period_metrics.value, width = 600, height = 500, label = "Prod",
                                                        xlim = (0,1), ylim = (0,1), grid = True) *  Slope(slope=1, y_intercept=0).opts(color='black', line_dash='dashed') * roc_baseline_p 
    #KS
    ks_b = baseline.groupby([period_metrics.value]).apply(ks)
    ks_p = prod.groupby([period_metrics.value]).apply(ks)
    baseline_ks = pn.widgets.DataFrame(ks_b)
    prod_ks = pn.widgets.DataFrame(ks_p,name = 'AUC') #autosize_mode='fit_columns'

    #LIFT
    baseline_lift_raw, baseline_lift_raw_bins = lift_init(df = baseline)
    baseline_lift_raw = baseline_lift_raw.rename(columns = {'Baseline': b_label})
    prod_lift_raw,  prod_lift_raw_bins = lift_init(df = prod, baseline = baseline, is_baseline = False)
    cols_b = baseline_lift_raw.columns.drop(['BINS', 'SCORE'])
    cols = prod_lift_raw.columns.drop(['BINS', 'SCORE'])

    baseline_lift = baseline_lift_raw.loc[baseline_lift_raw.BINS =='10',cols_b]
    prod_lift = prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols]
    # prod_lift = pd.concat([prod_lift.dropna(subset = [col]).dropna(axis = 1).reset_index(drop = 1) for col in prod_lift][1:], axis = 1)
    lift_table = prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols].melt(id_vars="SCORE_BAND", 
                                var_name='column', 
                                value_name='value').dropna().reset_index(drop = True).rename(columns = {'column':period_metrics.value , 'value': 'Target_PCT'})
    # print(prod_lift_raw_bins.loc[prod_lift_raw_bins.BINS ==10])
    lift_table = lift_table.hvplot.table(groupby = period_metrics.value, title="%s Lift Table"%period_metrics.value, hover = True, responsive=True, 
                shared_axes= False, fit_columns = True,
                padding=True , index_position = 0, fontscale = 1.5)
    # print(prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols])
    # print(baseline_lift_raw.loc[baseline_lift_raw.BINS == '10',cols_b])
    prod_lift_raw['BINS'] = prod_lift_raw['BINS'].astype(int)
    baseline_lift_raw['BINS'] = baseline_lift_raw['BINS'].astype(int)
    
    prod_lift_raw_bins['SCORE_BAND'] = prod_lift_raw_bins['SCORE_BAND'].astype(str)
    # prod_lift_raw_bins['BINS'] = prod_lift_raw_bins['BINS'].astype(str)

    baseline_lift_raw_bins['SCORE_BAND'] = baseline_lift_raw_bins['SCORE_BAND'].astype(str)
    # baseline_lift_raw_bins['BINS'] = baseline_lift_raw_bins['BINS'].astype(str)  

    # print(prod_lift_raw.loc[:,list(cols)+['BINS']])
    p1 = prod_lift_raw_bins.set_index('SCORE_BAND'
                                                      ).reset_index().hvplot.line(x = 'SCORE_BAND', groupby = ['BINS', period_metrics.value],
                                                          grid = True, width = 1200, height = 500,
                                                          label = 'Production', rot = 45)
    
    # print(baseline_lift_raw_bins)  
    # print(prod_lift_raw_bins)                                                    
    p2 = prod_lift_raw_bins.set_index('SCORE_BAND'
                                                        ).reset_index().hvplot.scatter(x = 'SCORE_BAND', groupby = ['BINS', period_metrics.value], grid = True, color='DarkBlue', label='Production', rot = 45) 

    b_label = baseline.MONTHLY.min()
    # print(baseline_lift_raw.loc[baseline_lift_raw.BINS == '10',cols_b][b_label])
    b1 = baseline_lift_raw_bins.hvplot.line(x = 'SCORE_BAND', groupby = ['BINS'],
                                                            grid = True, width = 1200, height = 500,
                                                            line_dash='dashed', color = 'black', label = b_label, rot = 45)

    b2 = baseline_lift_raw_bins.hvplot.scatter(x = 'SCORE_BAND', groupby = ['BINS'], grid = True, color='DarkGreen', label = b_label, rot = 45) 

    final_lift_plots = (p1*p2*b1*b2).opts(ylabel = '%target_rate_mean', title = "%s Lift Chart " % (period_metrics.value.title())) 

    #LABEL_DRIFT
    mean_score_prod = prod.groupby(period_metrics.value).agg(MEAN_SCORE=("SCORE","mean"), MEAN_TARGET=("TARGET","mean"),Count = ("TARGET","count"))
    mean_score_base = baseline.groupby(period_metrics.value).agg(MEAN_SCORE=("SCORE","mean"), MEAN_TARGET=("TARGET","mean"),Count = ("TARGET","count"))
    baseline_label_drift = pn.widgets.DataFrame(mean_score_base)
    prod_label_drift = pn.widgets.DataFrame(mean_score_prod,name = 'DRIFT')

    #Lift Tables
    # gains_final_all,_ = gains_table_proba(prod,'TARGET', 'SCORE')
    lift_data = pd.concat([baseline_lift, prod_lift], axis = 0)
    lift_data = pd.concat([lift_data.dropna(subset = [col]).dropna(axis = 1).reset_index(drop = 1) for col in lift_data][1:], axis = 1).dropna(axis = 1, how = 'any')
    lift_data = lift_data.loc[:,~lift_data.columns.duplicated()].set_index('SCORE_BAND')
    if (lift_data.shape[1] > 4) | (lift_data.shape[0] > 10):
        prod_lift = pn.pane.Markdown('### Please download the csv as the lift table will congest the screen')
    else:
        prod_lift = pn.widgets.DataFrame(lift_data,name = 'LIFT')
    #GAINS_TABLE
    gains_final_prod,_ = gains_table_proba(prod,'TARGET', 'SCORE')
    gains_final_base,_ = gains_table_proba(baseline,'TARGET', 'SCORE')
    gains_final_base.index.names = [b_label]
    gains_final_p = pn.widgets.DataFrame(gains_final_prod.set_index(['low','high']),name = 'GAINS',)
    gains_final_b = pn.widgets.DataFrame(gains_final_base.set_index(['low','high']),name = 'GAINS',)

    ece, bin_probamean, bin_ymean, bin_id, bin_count, bin_edges = expected_calibration_error(prod.TARGET.values, prod.SCORE.values)
    error = pd.DataFrame(np.array([bin_probamean, bin_ymean]).T,columns= ["SCORE_MEAN", "TARGET_MEAN"])
    error_plot = error.hvplot.scatter(x ='SCORE_MEAN', y = 'TARGET_MEAN', width = 800, height = 500, label = "Bin (Score vs Target Mean)", title = 'Model Scores Calibration (--- Perfect Calibration)',
                                                        xlim = (0,1), ylim = (0,1), grid = True, xlabel = 'Bins Mean of Scores', ylabel = 'Bins Mean of Target') *  Slope(slope=1, y_intercept=0,legend = 'Perfect Calibration').opts(color='black', line_dash='dashed')  
    variable_ = pn.pane.Alert('''### FJ Day Count: \n%s 
    '''%(datetime.datetime.now() - pd.Timestamp('2023-03-06 03:27')), alert_type="success")
    return pn.Tabs(
        ('Metrics', pn.Column(
                    pn.Row(intiate, intiate2, intiate3, width = 1200),
                    '# PSI',
                    pn.Row(psi_results, save_csv(psi_, 'PSI')),
                    '# AUC',
                    pn.Row(prod_auc, baseline_auc, save_csv(pd.concat([auc_b, auc_p], axis = 0), 'AUC')),
                    '# KS',
                    pn.Row(prod_ks, baseline_ks, save_csv(pd.concat([ks_b, ks_p], axis = 0), 'KS')),
                    '# LABEL DRIFT',
                    pn.Row(prod_label_drift, baseline_label_drift, save_csv(pd.concat([mean_score_base, mean_score_prod], axis = 0), 'LABEL_DRIFT')),
                    '# LIFT TABLES',
                    pn.Row(prod_lift, save_csv(lift_data, 'LIFT_TABLES')),
                    '# GAINS TABLE',
                    pn.Row(gains_final_b, gains_final_p, save_csv(pd.concat([gains_final_base, gains_final_prod], axis = 1), 'GAINS_TABLES')),
                    get_xlsx(psi_, pd.concat([auc_b, auc_p], axis = 0), pd.concat([ks_b, ks_p], axis = 0), pd.concat([mean_score_base, mean_score_prod], axis = 0), lift_data, pd.concat([gains_final_base, gains_final_prod], axis = 1)), 
                    pn.Row(variable_, width = 200),
                             )
        ), #sizing_mode='stretch_width'
        ('Charts', pn.Column(pn.Row(roc_plot.opts(legend_position = 'bottom_right'), error_plot.opts(legend_position = 'top_left')) ,
                             lift_table,
                             final_lift_plots.opts(legend_position = 'bottom_right')
                            )
        )
    
    )
        

    # return pn.Tabs(
    #         ('Analysis', pn.Column(
    #             pn.Row(vol_ret, pn.layout.Spacer(width=20), pn.Column(div, table), sizing_mode='stretch_width'),
    #             pn.Column(pn.Row(year, investment), return_curve, sizing_mode='stretch_width'),
    #             sizing_mode='stretch_width')),
    #         ('Timeseries', timeseries),
    #         ('Log Return', pn.Column(
    #             '## Daily normalized log returns',
    #             'Width of distribution indicates volatility and center of distribution the mean daily return.',
    #             log_ret_hists,
    #             sizing_mode='stretch_width'
    #         ))
    #     )

pn.Row(pn.Column(widgets), pn.layout.Spacer(width=30), run).servable()













# Caveats
# The maximum sizes set in either Bokeh or Tornado refer to the maximum size of the message that 
# is transferred through the web socket connection, which is going to be larger than the actual 
# size of the uploaded file since the file content is encoded in a base64 string. So if you set a
# maximum size of 100 MB for your application, you should indicate your users that the upload
# limit is a value that is less than 100 MB.

# When a file whose size is larger than the limits is selected by a user, their browser/tab may
# just crash. Alternatively the web socket connection can close (sometimes with an error message
# printed in the browser console such as [bokeh] Lost websocket 0 connection, 1009 (message too 
# big)) which means the application will become unresponsive and needs to be refreshed.

# app = ...

# MAX_SIZE_MB = 150

# pn.serve(
#     app,
#     # Increase the maximum websocket message size allowed by Bokeh
#     websocket_max_message_size=MAX_SIZE_MB*1024*1014,
#     # Increase the maximum buffer size allowed by Tornado
#     http_server_kwargs={'max_buffer_size': MAX_SIZE_MB*1024*1014}
# )