Update app.py

app.py

@@ -1,147 +1,784 @@
+#!/usr/bin/env python
+# coding: utf-8
+import numpy as np
 import io
-import random
-from typing import List, Tuple
-
-import aiohttp
+import sys
+import os
+import pandas as pd
+import datetime
+import gc #garabage collector
+from io import BytesIO
 import panel as pn
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor
+import holoviews as hv
+import hvplot.pandas
+import xlsxwriter
+from warnings import filterwarnings
+'''
+development env: panel serve script.py --autoreload
+prod prep: panel convert script.py --to pyodide-worker --out pyodide
+'''
 
-pn.extension(design="bootstrap", sizing_mode="stretch_width")
+filterwarnings("ignore")
+# hv.extension('bokeh')
+pn.extension( "plotly", template="fast")
 
-ICON_URLS = {
-    "brand-github": "https://github.com/holoviz/panel",
-    "brand-twitter": "https://twitter.com/Panel_Org",
-    "brand-linkedin": "https://www.linkedin.com/company/panel-org",
-    "message-circle": "https://discourse.holoviz.org/",
-    "brand-discord": "https://discord.gg/AXRHnJU6sP",
-}
+pn.state.template.param.update(
+    # site_url="",
+    site="ModelMonitor",
+    title="Classification Model Metrics",
+    # favicon="https://raw.githubusercontent.com/firobeid/firobeid.github.io/main/docs/compose-plots/Resources/favicon.ico",
+)
+#######################
+###UTILITY FUNCTIONS###
+#######################
+def percentage(df):
+    def segment(df):
+        return round(df["Count"]/df["Count"].sum(),4)
+    df["percent"] = segment(df)
+    return df
 
+def AUC(group):
+    from sklearn.metrics import roc_auc_score
+    auc = roc_auc_score(group['TARGET'],group['SCORE'])
+    # N = sum(group["N"])
+    N = round(len(group.loc[group["TARGET"].notna()]),0)
+    cols = ["AUC","Count"]
+    # return trapezoidal_rule(FPR.to_numpy(),TPR.to_numpy())
+    return pd.Series([auc, N], index = cols)
 
-async def random_url(_):
-    pet = random.choice(["cat", "dog"])
-    api_url = f"https://api.the{pet}api.com/v1/images/search"
-    async with aiohttp.ClientSession() as session:
-        async with session.get(api_url) as resp:
-            return (await resp.json())[0]["url"]
+def ROC(group):
+    from sklearn.metrics import roc_curve
+    FPR,TPR,T = roc_curve(group['TARGET'],group['SCORE'])
+    cols = ['TPR', 'FPR']
+    return pd.concat([pd.Series(TPR),pd.Series(FPR)], keys = cols, axis = 1)
 
+def ks(group):
+    from scipy.stats import ks_2samp
+    y_real = group['TARGET']
+    y_proba = group['SCORE']
+    
+    df = pd.DataFrame()
+    df['real'] = y_real
+    df['proba'] = y_proba
+    
+    # Recover each class
+    class0 = df[df['real'] == 0]
+    class1 = df[df['real'] == 1]
+    
+    ks_ = ks_2samp(class0['proba'], class1['proba'])
+    
+    N = round(len(group.loc[group["TARGET"].notna()]),0)
+    cols = ["KS","Count"]
+    
+    return pd.Series([ks_[0], N], index = cols)
 
-@pn.cache
-def load_processor_model(
-    processor_name: str, model_name: str
-) -> Tuple[CLIPProcessor, CLIPModel]:
-    processor = CLIPProcessor.from_pretrained(processor_name)
-    model = CLIPModel.from_pretrained(model_name)
-    return processor, model
+def psi(df):
+    '''
+    https://mwburke.github.io/data%20science/2018/04/29/population-stability-index.html#:~:text=To%20calculate%20the%20PSI%20we,the%20percents%20in%20each%20bucket.
+    '''
+    df[df == 0] = 0.001
+    sub = df.copy()
+    sub = sub.iloc[:,:-1].sub(df.validation,axis = 0)
+    div = df.copy()
+    div= div.iloc[:,:-1].div(df.validation, axis=0)
+    div = np.log(div)
+    return (sub*div).sum(axis = 0)
 
+def add_extremes_OOT(df, name:str, score:str):
+    '''
+    Mitigate bias in OOT/Serving/baseline set that might not have high confidence scores or low confidence scores
+    :param: name: str, name of the appid column
+    :param: score: str, name of the score column
+    '''
+    # df.loc[len(df.index)] = [np.nan, "Extreme_Case_Max", np.nan, np.nan, np.nan,994.0,0.0009,np.nan,np.nan,np.nan,np.nan]
+    # df.loc[len(df.index)] = [np.nan, "Extreme_Case_Min", np.nan, np.nan, np.nan,158.0,0.9999,np.nan,np.nan,np.nan,np.nan]
+    df.loc[len(df.index)] = [np.nan for i in range(0,df.shape[1])]
+    df.loc[(len(df.index)-1), [name, score]] = ["Extreme_Case_Max", 0.0009]
+    df.loc[len(df.index)] = [np.nan for i in range(0,df.shape[1])]
+    df.loc[(len(df.index)-1), [name, score]] = ["Extreme_Case_Min", 0.9999]
+    return df
 
-async def open_image_url(image_url: str) -> Image:
-    async with aiohttp.ClientSession() as session:
-        async with session.get(image_url) as resp:
-            return Image.open(io.BytesIO(await resp.read()))
+# def last_3months(df):
+#     from datetime import datetime
+#     from dateutil.relativedelta import relativedelta
+#     from pandas.tseries.offsets import MonthEnd
 
+#     end_of_month = ((pd.Timestamp(datetime.now().strftime('%Y-%m-%d')) - pd.Timedelta(70, unit='D')) + relativedelta(months=-1)) + MonthEnd(0)
+#     start_of_month = end_of_month + MonthEnd(-3) + relativedelta(days=1)
+#     end_of_month = end_of_month +relativedelta(hours=23, minutes=59, seconds=59)
+#     print('Start Month %r --- End Month %r' % (start_of_month, end_of_month))
+#     try:
+#         date_column = list(filter(lambda x:x.endswith("DATE"),gains_df.columns))[0]
+#     except:
+#         date_column = 'CREATED_DATE'
+#     return df[df[date_column].between(start_of_month, end_of_month)]
 
-def get_similarity_scores(class_items: List[str], image: Image) -> List[float]:
-    processor, model = load_processor_model(
-        "openai/clip-vit-base-patch32", "openai/clip-vit-base-patch32"
-    )
-    inputs = processor(
-        text=class_items,
-        images=[image],
-        return_tensors="pt",  # pytorch tensors
-    )
-    outputs = model(**inputs)
-    logits_per_image = outputs.logits_per_image
-    class_likelihoods = logits_per_image.softmax(dim=1).detach().numpy()
-    return class_likelihoods[0]
+def gains_table_proba(data=None,target=None, prob=None):
+    data = data.copy()
+    data['target0'] = 1 - data[target]
+    data['bucket'] = pd.qcut(data[prob], 10)
+    grouped = data.groupby('bucket', as_index = False)
+    kstable = pd.DataFrame()
+    kstable['min_prob'] = grouped.min()[prob]
+    kstable['max_prob'] = grouped.max()[prob]
+    kstable['count'] = grouped.count()['target0']
+    kstable['cum_total']=(kstable['count'] / kstable['count'].sum()).cumsum()
+    kstable['events']  = grouped.sum()[target]
+    kstable['nonevents'] = grouped.sum()['target0']
+    kstable['interval_rate'] = kstable['events'] / kstable['count']
+    kstable = kstable.sort_values(by="min_prob", ascending=0).reset_index(drop = True)
+    kstable['event_rate'] = (kstable.events / data[target].sum()).apply('{0:.2%}'.format)
+    kstable['nonevent_rate'] = (kstable.nonevents / data['target0'].sum()).apply('{0:.2%}'.format)
+    kstable['cum_eventrate']=(kstable.events / data[target].sum()).cumsum()
+    kstable['cum_noneventrate']=(kstable.nonevents / data['target0'].sum()).cumsum()
+    kstable['mid_point'] = np.nan
+    kstable['KS'] = np.round(kstable['cum_eventrate']-kstable['cum_noneventrate'], 4) * 100
 
+    #Formating
+    kstable["cum_total"] = kstable["cum_total"].sort_values().values
+    kstable = kstable.rename(columns={"min_prob":"low", "max_prob":"high"})
+    kstable['mid_point'] = round((kstable['high'] + kstable['low']) / 2, 4)
+    kstable['cum_eventrate']= kstable['cum_eventrate'].apply('{0:.2%}'.format)
+    kstable['cum_noneventrate']= kstable['cum_noneventrate'].apply('{0:.2%}'.format)
+    kstable.index = range(1,11)
+    kstable.index.rename('Decile', inplace=True)
+    pd.set_option('display.max_columns', 15)
+    # print(kstable)
+    #Display KS
+    from colorama import Fore
+    ks_3mnths = "KS is " + str(max(kstable['KS']))+"%"+ " at decile " + str((kstable.index[kstable['KS']==max(kstable['KS'])][0]))
+    print("KS is " + str(max(kstable['KS']))+"%"+ " at decile " + str((kstable.index[kstable['KS']==max(kstable['KS'])][0])))
+    kstable['cum_eventrate']= kstable['cum_eventrate'].str.replace("%","").astype(float)
+    kstable['cum_noneventrate']= kstable['cum_noneventrate'].str.replace("%","").astype(float)
+    kstable.index = list(range(10,0,-1))
+    kstable = kstable.iloc[::-1]
+    return(kstable, ks_3mnths)
 
-async def process_inputs(class_names: List[str], image_url: str):
-    """
-    High level function that takes in the user inputs and returns the
-    classification results as panel objects.
-    """
-    try:
-        main.disabled = True
-        if not image_url:
-            yield "##### ⚠️ Provide an image URL"
-            return
+def calculate_psi(expected, actual, buckettype='bins', buckets=10, axis=0):
+    # https://www.kaggle.com/code/podsyp/population-stability-index
+    '''Calculate the PSI (population stability index) across all variables
+    Args:
+       expected: numpy matrix of original values
+       actual: numpy matrix of new values, same size as expected
+       buckettype: type of strategy for creating buckets, bins splits into even splits, quantiles splits into quantile buckets
+       buckets: number of quantiles to use in bucketing variables
+       axis: axis by which variables are defined, 0 for vertical, 1 for horizontal
+    Returns:
+       psi_values: ndarray of psi values for each variable
+    Author:
+       Matthew Burke
+       github.com/mwburke
+       worksofchart.com
+    '''
+
+    def psi(expected_array, actual_array, buckets):
+        '''Calculate the PSI for a single variable
+        Args:
+           expected_array: numpy array of original values
+           actual_array: numpy array of new values, same size as expected
+           buckets: number of percentile ranges to bucket the values into
+        Returns:
+           psi_value: calculated PSI value
+        '''
+
+        def scale_range (input, min, max):
+            input += -(np.min(input))
+            input /= np.max(input) / (max - min)
+            input += min
+            return input
+
+
+        breakpoints = np.arange(0, buckets + 1) / (buckets) * 100
+
+        if buckettype == 'bins':
+            breakpoints = scale_range(breakpoints, np.min(expected_array), np.max(expected_array))
+        elif buckettype == 'quantiles':
+            breakpoints = np.stack([np.percentile(expected_array, b) for b in breakpoints])
+
+
+
+        expected_percents = np.histogram(expected_array, breakpoints)[0] / len(expected_array)
+        actual_percents = np.histogram(actual_array, breakpoints)[0] / len(actual_array)
+
+        def sub_psi(e_perc, a_perc):
+            '''Calculate the actual PSI value from comparing the values.
+               Update the actual value to a very small number if equal to zero
+            '''
+            if a_perc == 0:
+                a_perc = 0.0001
+            if e_perc == 0:
+                e_perc = 0.0001
+
+            value = (e_perc - a_perc) * np.log(e_perc / a_perc)
+            return(value)
+
+        psi_value = np.sum(sub_psi(expected_percents[i], actual_percents[i]) for i in range(0, len(expected_percents)))
+
+        return(psi_value)
+
+    if len(expected.shape) == 1:
+        psi_values = np.empty(len(expected.shape))
+    else:
+        psi_values = np.empty(expected.shape[axis])
+
+    for i in range(0, len(psi_values)):
+        if len(psi_values) == 1:
+            psi_values = psi(expected, actual, buckets)
+        elif axis == 0:
+            psi_values[i] = psi(expected[:,i], actual[:,i], buckets)
+        elif axis == 1:
+            psi_values[i] = psi(expected[i,:], actual[i,:], buckets)
+
+    return(psi_values)
+
+    return round(10 **((158.313177 - UW5_Score) /274.360149), 18)
+
+def lift_init(df:pd.DataFrame, baseline = None, is_baseline = True):
+    from tqdm import tqdm
+    # global standalone_scores_OOT
+    cols = ['SCORE']
     
-        yield "##### ⚙ Fetching image and running model..."
+    lift_chart_data_OOT = pd.DataFrame()
+    for q in tqdm([10,20,50,100]):
+        # df_new["QUARTER"] = pd.PeriodIndex(df_new.CREATE_DATE, freq='Q')
+        # fd = baseline.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, ), axis = 0).copy()
+        # pd.cut(prod['SCORE'], bins = pd.qcut(baseline['SCORE'],10, retbins = True)[1]) 
+        if is_baseline == True:
+            # print(df)
+            # print(df.dropna(subset = ['MONTHLY']))
+            fd = df.dropna(subset = [period_metrics.value])[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]) , axis = 0).copy()
+            fd = pd.concat([df.dropna(subset = [period_metrics.value])[period_metrics.value], df.dropna(subset = [period_metrics.value])['TARGET'], fd], axis = 1)
+            fd = pd.concat([fd.groupby(x)['TARGET'].mean().fillna(0) for x in fd[cols]], axis = 1, keys = cols)
+            fd.index.name = 'SCORE_BAND'
+            
+        else:
+            # print(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel().shape)
+            # print(pd.qcut(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel(),q=q, retbins = True))
+            bins_ = pd.qcut(baseline.dropna(subset = [period_metrics.value])[cols].values.ravel(),q=q, retbins = True)[1]
+            fd = df.groupby([period_metrics.value]).apply(lambda col: col[cols].apply(lambda col: pd.cut(col, bins = bins_), axis = 0)).copy()
+            # fd = df.groupby(period_metrics.value).apply(lambda col: col[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]), axis = 0)).copy()
+            fd = pd.concat([df[period_metrics.value], df['TARGET'], fd], axis = 1)
+            fd = fd.groupby(period_metrics.value).apply(lambda col: pd.concat([col.groupby(x)['TARGET'].mean().fillna(0) for x in col[cols]], axis = 1, keys = cols))
+            fd.index.names = [period_metrics.value, 'SCORE_BAND']
+        # fd['APPLICATION_MONTH'] = fd['APPLICATION_MONTH'].astype(str)
+        fd = fd.reset_index()
+        fd['BINS'] = q
+        lift_chart_data_OOT = lift_chart_data_OOT.append(fd)
+    if is_baseline == True:
+        lift_chart_data_OOT[period_metrics.value] = 'Baseline'
+       
+    standalone_scores_OOT = lift_chart_data_OOT.melt(id_vars=[period_metrics.value,'BINS','SCORE_BAND'],value_vars=cols,
+                                        var_name='SCORE', 
+                                        value_name='BAD_RATE').dropna().reset_index(drop = True).copy()
+    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(str)
+    standalone_scores_OOT = pd.concat([standalone_scores_OOT["BINS"] + "-" + standalone_scores_OOT["SCORE_BAND"] + "-" + standalone_scores_OOT["SCORE"], 
+                                       standalone_scores_OOT[[period_metrics.value,'BAD_RATE']]], axis = 1).rename(columns = {0:'BINS_SCOREBAND_SCORE'})
+    standalone_scores_OOT = standalone_scores_OOT.pivot(index = 'BINS_SCOREBAND_SCORE', columns=period_metrics.value)['BAD_RATE'].reset_index()
+    standalone_scores_OOT.index.name = ""
+    standalone_scores_OOT.columns.name = ""
+    standalone_scores_OOT = pd.concat([standalone_scores_OOT['BINS_SCOREBAND_SCORE'].str.split('-', expand=True),
+                                   standalone_scores_OOT],axis = 1).rename(columns ={0:'BINS', 1: 'SCORE_BAND', 2: 'SCORE'}).drop(columns = 'BINS_SCOREBAND_SCORE')
+    # standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']]#.astype(int)
+    standalone_scores_OOT['BINS'] = standalone_scores_OOT['BINS']
+    standalone_scores_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)
+    return standalone_scores_OOT, lift_chart_data_OOT
+
+def lift_init_plots(df:pd.DataFrame, is_baseline = True):
+    from tqdm import tqdm
+    # global standalone_scores_OOT
+    cols = ['SCORE']
+
+    lift_chart_data_OOT = pd.DataFrame()
+    for q in tqdm([10,20,50,100]):
+        # df_new["QUARTER"] = pd.PeriodIndex(df_new.CREATE_DATE, freq='Q')
+        # fd = baseline.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, ), axis = 0).copy()
+        # pd.cut(prod['SCORE'], bins = pd.qcut(baseline['SCORE'],10, retbins = True)[1]) 
+        # fd = df.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.cut(col, bins = pd.qcut(col,q=q, retbins = True)[1]) , axis = 0).copy()
+        if is_baseline == True:
+            fd = df.dropna(subset = period_metrics.value)[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, labels=range(1, q + 1)), axis = 0).copy()
+            fd = pd.concat([df.dropna(subset = period_metrics.value)[period_metrics.value], df.dropna(subset = period_metrics.value)['TARGET'], fd], axis = 1)
+            fd = pd.concat([fd.groupby(x)['TARGET'].mean().fillna(0) for x in fd[cols]], axis = 1, keys = cols)
+            fd.index.name = 'SCORE_BAND'
+            
+        else:
+            fd = df.groupby(period_metrics.value).apply(lambda col: col[cols].apply(lambda col: pd.qcut(col.rank(method='first'),q = q, labels=range(1,q + 1)), axis = 0)).copy()
+            fd = pd.concat([df[period_metrics.value], df['TARGET'], fd], axis = 1)
+            fd = fd.groupby(period_metrics.value).apply(lambda col: pd.concat([col.groupby(x)['TARGET'].mean().fillna(0) for x in col[cols]], axis = 1, keys = cols))
+            # print(fd.index)
+            fd.index.names = [period_metrics.value, 'SCORE_BAND']
+            # fd = fd.reset_index(names = ['APPLICATION_MONTH', 'SCORE_BAND'])  
+        fd = fd.reset_index()
+        # fd['APPLICATION_MONTH'] = fd['APPLICATION_MONTH'].astype(str)
+        fd['BINS'] = q
+        lift_chart_data_OOT = lift_chart_data_OOT.append(fd)
+    if is_baseline == True:
+        lift_chart_data_OOT[period_metrics.value] = 'Baseline'
+    lift_chart_data_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)   
+    standalone_scores_OOT = lift_chart_data_OOT.melt(id_vars=[period_metrics.value,'BINS','SCORE_BAND'],value_vars=cols,
+                                        var_name='SCORE', 
+                                        value_name='BAD_RATE').dropna().reset_index(drop = True).copy()
+    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(str)
+    standalone_scores_OOT = pd.concat([standalone_scores_OOT["BINS"] + "-" + standalone_scores_OOT["SCORE_BAND"] + "-" + standalone_scores_OOT["SCORE"], 
+                                       standalone_scores_OOT[[period_metrics.value,'BAD_RATE']]], axis = 1).rename(columns = {0:'BINS_SCOREBAND_SCORE'})
+    standalone_scores_OOT = standalone_scores_OOT.pivot(index = 'BINS_SCOREBAND_SCORE', columns=period_metrics.value)['BAD_RATE'].reset_index()
+    standalone_scores_OOT.index.name = ""
+    standalone_scores_OOT.columns.name = ""
+    standalone_scores_OOT = pd.concat([standalone_scores_OOT['BINS_SCOREBAND_SCORE'].str.split('-', expand=True),
+                                   standalone_scores_OOT],axis = 1).rename(columns ={0:'BINS', 1: 'SCORE_BAND', 2: 'SCORE'}).drop(columns = 'BINS_SCOREBAND_SCORE')
+    standalone_scores_OOT[['BINS', 'SCORE_BAND']] = standalone_scores_OOT[['BINS', 'SCORE_BAND']].astype(int)
+    standalone_scores_OOT['BINS'] = standalone_scores_OOT['BINS']
+    standalone_scores_OOT.sort_values(['SCORE', 'SCORE_BAND'], inplace = True)
+    return standalone_scores_OOT
+
+def save_csv(df, metric):
+    from io import StringIO
+    sio = StringIO()
+    df.to_csv(sio)
+    sio.seek(0)
+    return pn.widgets.FileDownload(sio, embed=True, filename='%s.csv'%metric)
+
+def get_xlsx(df1,df2,df3,df4,df5,df6):
+    from io import BytesIO
+    output = BytesIO()
+    writer = pd.ExcelWriter(output, engine='xlsxwriter')
+    df1.to_excel(writer, sheet_name="PSI")
+    df2.to_excel(writer, sheet_name="AUC")
+    df3.to_excel(writer, sheet_name="KS")
+    df4.to_excel(writer, sheet_name="LABEL_DRIFT")
+    df5.to_excel(writer, sheet_name="LABEL_Tables")
+    df6.to_excel(writer, sheet_name="GAINS_Tables")
+    writer.save() # Important!
+    output.seek(0) # Important!
+    return pn.widgets.FileDownload(output,embed=True, filename='results.csv', button_type="primary")
+
+
+def expected_calibration_error(y, proba, bins = 'fd'):
+  import numpy as np
+  bin_count, bin_edges = np.histogram(proba, bins = bins)
+  n_bins = len(bin_count)
+  bin_edges[0] -= 1e-8 # because left edge is not included
+  bin_id = np.digitize(proba, bin_edges, right = True) - 1
+  bin_ysum = np.bincount(bin_id, weights = y, minlength = n_bins)
+  bin_probasum = np.bincount(bin_id, weights = proba, minlength = n_bins)
+  bin_ymean = np.divide(bin_ysum, bin_count, out = np.zeros(n_bins), where = bin_count > 0)
+  bin_probamean = np.divide(bin_probasum, bin_count, out = np.zeros(n_bins), where = bin_count > 0)
+  ece = np.abs((bin_probamean - bin_ymean) * bin_count).sum() / len(proba)
+  return ece, bin_probamean, bin_ymean, bin_id, bin_count, bin_edges
+###############################
+###END OFF UTILITY FUNCTIONS###
+###############################
+
+text = """
+#Classification Model Metrics
+## AUTHOR: [`FIRAS ALI OBEID`](https://www.linkedin.com/in/feras-obeid/) 
+###  GNU General Public License v3.0 (GPL-3.0)
+#### Developed while working at [OppFi Inc.](https://www.oppfi.com/)
+
+This tool performs ML model ,in production, monitoring across time, 
+where production weeks/months/quarters are compared too a selective baseline.
+
+1. Upload a CSV containing:
+
+**(Date)** Highly Recommended but **optional** 
+**(Score)** Probability Predictions  
+**(Target)** Binary Target/True Label
+ 
+2. Check the box if you CSV has a DATE column, otherwise dates are generated based on current timestamp and spanning back by 
+timedelta of csv length in hourly frequency.
+
+3. Choose & press the right columns in the `Select Boxes` below when you upload a csv
+
+4. Select a baseline date slice **mandatory**. If your baseline is from a different time then the production time,
+make sure to append it to the csv before uploading.
+
+5. Press Get Metrics
+
+6. Wait few seconds and analyze the updated charts
+"""
+
+
+
+# date = str(input('What is the name off the date column: ').upper())
+# id_ = str(input('What is the name off the APP name/ID column: ').upper())
+# score = str(input('What is the name off the score column (i.e UW5,DM_QL...): ').upper())
+# target = str(input('What is the name off the Target column (i.e Real target values such as PD70_RATIO...: ').upper())
+
+file_input = pn.widgets.FileInput(align='center')
+date_selector = pn.widgets.Select(name='Select Date Column',)
+check_date = pn.widgets.Checkbox(name = '<--',value = False) # T/F
+target_selector = pn.widgets.Select(name='Select Target Variable(True Label)')
+score_selector = pn.widgets.Select(name='Select Predictions Column(Raw Probaility)')
+period_metrics = pn.widgets.Select(name='Select Period', options = ['MONTHLY','WEEKLY', 'QUARTERLY'])
+
+date_range_ = pn.widgets.DateRangeSlider(name='Baseline Period',) #value=(start, end), start=start, end=end
+
+random_seed = pn.widgets.IntSlider(name='Random Seed for Random Generated Data (OnSet)', value=42, start=0, end=1000, step=1)
+
+button = pn.widgets.Button(name='Get Metrics')
+widgets = pn.WidgetBox(
+    pn.panel(text, margin=(0, 20)),
+    pn.panel('**Check box if your data has a date column *before uploading the file* \n (otherwise keep it empty)**'),
+    check_date,
+    file_input,
+    random_seed,
+    pn.panel('\n'),
+    date_selector,
+    target_selector,
+    score_selector,
+    period_metrics,
+    date_range_,
+    button
+)
+
+# start, end = stocks.index.min(), stocks.index.max()
+# year = pn.widgets.DateRangeSlider(name='Year', value=(start, end), start=start, end=end)
+# ,id_:'ID',
+
+
+def get_data():
+    global df
+    if file_input.value is None:
+        np.random.seed(random_seed.value)
         try:
-            pil_img = await open_image_url(image_url)
-            img = pn.pane.Image(pil_img, height=400, align="center")
-        except Exception as e:
-            yield f"##### 😔 Something went wrong, please try a different URL!"
-            return
+            df = pd.DataFrame({'DATE': pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = 9999)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H"),
+                            'ID': [i for i in range(10000)],
+                            'SCORE':np.random.uniform(size = 10000),
+                            'TARGET': np.random.choice([0,1],10000, p=[0.9,0.1])})
+        except:
+            try:
+                df = pd.DataFrame({'DATE': pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = 9999 + 1)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H"),
+                                'ID': [i for i in range(10000)],
+                                'SCORE':np.random.uniform(size = 10000),
+                                'TARGET': np.random.choice([0,1],10000, p=[0.9,0.1])})  
+            except:
+                df = pd.DataFrame({'DATE': pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = 9999 - 1)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H"),
+                                'ID': [i for i in range(10000)],
+                                'SCORE':np.random.uniform(size = 10000),
+                                'TARGET': np.random.choice([0,1],10000, p=[0.9,0.1])})            
+        # df.to_csv("test_upload.csv")
+    else:
+        df = BytesIO()
+        df.write(file_input.value)
+        df.seek(0)
+        try:
+                df = pd.read_csv(df, error_bad_lines=False).apply(pd.to_numeric, errors='ignore')
+        except:
+                df = pd.read_csv(df, error_bad_lines=False)
+
+        df = df.select_dtypes(exclude=["category"])
+        df = df.replace([np.inf, -np.inf], np.nan)
+        df.columns = [i.upper() for i in df.columns]
+    return df
+
+def update_target(event):
+    df = get_data()
+    cols = list(df.columns)
+    date_selector.set_param(options=cols)
+    target_selector.set_param(options=cols)
+    score_selector.set_param(options=cols)
+    # print(check_date.value)
+    # print(type(df.DATE.min()))
+    if check_date.value == True:
+        date_column = [i.find("DATE") for i in df.columns] 
+        date_column = [date_column.index(i) for i in [i for i in date_column if i !=-1]]
+        if len(date_column) > 0:
+            df = df.iloc[:,date_column].iloc[:,[0]]
+            df.columns = ['DATE']
+            print(type(df.DATE.min()))
+            start, end = pd.Timestamp(df.DATE.min()),  pd.Timestamp(df.DATE.max())
+            try:
+                date_range_.set_param(value=(start, end), start=start, end=end)
+            except:
+                date_range_.set_param(value=(end, start), start=end, end=start)
+        else:
+            print('Creating synthetic dates')
+            synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df))), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H") #remove len(df) - 1
+            df['DATE'] = synthetic_date[:len(df)]
+            start, end = df.DATE.min(), df.DATE.max()
+            date_range_.set_param(value=(start, end), start=start, end=end)
+    else:
+        print('Creating synthetic dates')
+        synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df))), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H")
+        df['DATE'] = synthetic_date[:len(df)]
+        start, end = df.DATE.min(), df.DATE.max()
+        date_range_.set_param(value=(start, end), start=start, end=end)
+file_input.param.watch(update_target, 'value')
+update_target(None)
+
+@pn.depends(button.param.clicks)
+def run(_):
+    print(random_seed.value)
+    print(score_selector.value)
+    df = get_data()
+    try:
+        if file_input.value is None:
+            pass
+        elif check_date.value == True:
+            df = df.rename(columns={date_selector.value:'DATE',score_selector.value:'SCORE',target_selector.value:'TARGET'})
+        else:
+            synthetic_date = pd.date_range(start = (datetime.datetime.today() - pd.DateOffset(hours = len(df) - 1)), end = datetime.datetime.today(), tz = "US/Eastern", freq = "H")
+            df['DATE'] = synthetic_date[:len(df)]
+            df = df.rename(columns={score_selector.value:'SCORE',target_selector.value:'TARGET'})
+    except Exception as e:
+        return pn.pane.Markdown(f"""{e}""")
+    try:
+        df.DATE = pd.to_datetime(df.DATE, format="%Y-%m-%d %H:%M:%S", utc = True)
+        # print(pd.to_datetime(df.DATE,utc = True))
+        df["MONTHLY"] = df["DATE"].dt.strftime('%Y-%m')
+        print(f"J - DAYS COUNT: {datetime.datetime.now() - pd.Timestamp('2023-03-06 03:27')}" )
+        df['QUARTERLY'] = pd.PeriodIndex(df.DATE, freq='Q').astype(str)
+        df['WEEKLY'] = pd.PeriodIndex(df.DATE, freq='W').astype(str)
+    except Exception as e:
+        return pn.pane.Markdown(f"""{e}""")    
+    df = df.reset_index().rename(columns={df.index.name:'ID'}) #crate synthetic prediction ID for my code to run
+    # df = df.dropna(subset = 'TARGET', axis = 1)
+    df = df[~(df.TARGET.isna()) | (df.SCORE.isna())]
+    if df.TARGET.nunique() > 2:
+        df.TARGET = np.where(df.TARGET > 0 , 1 , 0)      
+    df.SCORE = df.SCORE.astype(np.float64)
+
     
-        class_items = class_names.split(",")
-        class_likelihoods = get_similarity_scores(class_items, pil_img)
+
+    # baselines
+    # try:
+    #     baseline = df.set_index('MONTHLY').loc[date_range_.value[0]: date_range_.value[1]].reset_index().copy()
+    # except:
+    #     baseline = df.copy()
+    #     baseline = baseline.set_index('MONTHLY')
+    #     baseline.index = pd.to_datetime(baseline.index)
+    #     baseline = baseline.loc[date_range_.value[0]: date_range_.value[1]].reset_index()
+    #     baseline["MONTHLY"] = baseline["MONTHLY"] .dt.strftime('%Y-%m')
+    print(date_range_.value[0])
+    print(date_range_.value[1])
+
+    baseline = df.set_index('DATE').sort_index().loc[date_range_.value[0]: date_range_.value[1]].reset_index()
+    print(baseline.DATE.min())
+    print(baseline.DATE.max())
+    print(df.DATE.max())
+
+    # print(df.set_index('DATE').loc[date_range_.value[0]: date_range_.value[1]].index.max())
+    #prods
+    # prod = df.loc[~df.MONTHLY.isin(list(baseline.MONTHLY.unique()))].copy()
+    prod_dates = df.set_index('DATE').sort_index().index.difference(baseline.set_index('DATE').index)
+    # print(prod_dates)
+    prod = df.set_index('DATE').loc[prod_dates].reset_index()
+    if len(baseline) > len(prod):
+        prod = baseline
+    ##START##
+    intiate = pn.pane.Alert('''### Baseline Period: \n%s to %s
+    '''%(baseline.DATE.min(),baseline.DATE.max()), alert_type="info")
+    intiate2 = pn.pane.Alert('''### Production Period: \n%s to %s
+    '''%(prod.DATE.min(),prod.DATE.max()), alert_type="info")
+    if prod.equals(baseline):
+        intiate3 = pn.pane.Alert('''### Baseline Set is identical to Production Set \n Please choose a slice to be a baseline set''', alert_type="danger")
+    else:
+        intiate3 = None
+    ##PSI##
+    baseline_psi = baseline.copy()
+    prod_psi = prod.copy()
+
+    baseline_psi = add_extremes_OOT(baseline_psi, name = 'ID', score = 'SCORE')
+    prod_psi["DEC_BANDS"] = pd.cut(prod_psi['SCORE'], bins = pd.qcut(baseline_psi['SCORE'],10, retbins = True)[1]) 
+    prod_psi = prod_psi.groupby([period_metrics.value,
+                                    "DEC_BANDS"]).agg(Count = ("DEC_BANDS",
+                                    "count")).sort_index(level = 0).reset_index()
+    prod_psi = prod_psi.groupby(period_metrics.value).apply(percentage).drop("Count",axis = 1)
+
+    baseline_psi["DECILE"] = pd.cut(baseline_psi['SCORE'], bins = pd.qcut(baseline_psi['SCORE'],10, retbins = True)[1]) 
+    baseline_psi = baseline_psi["DECILE"].value_counts()
+    baseline_psi = baseline_psi / sum(baseline_psi)
+    baseline_psi = baseline_psi.reset_index().rename(columns={"index":"DEC_BANDS", "DECILE": "percent"})
+    baseline_psi[period_metrics.value] = "validation"
+    baseline_psi = baseline_psi[[period_metrics.value, "DEC_BANDS", "percent"]]
+
+    prod_psi = pd.concat([prod_psi,baseline_psi])
+
+    prod_psi = prod_psi.pivot(index = "DEC_BANDS", columns=period_metrics.value)["percent"]
+    if len(baseline) < len(prod):
+        psi_ = psi(prod_psi).to_frame("%s_PSI"%period_metrics.value)
+        psi_results = pn.widgets.DataFrame(psi_)
+    else: 
+        psi_ = pd.DataFrame()
+        psi_results = pn.pane.Alert("### Choose a Baseline in the left banner to get PSI results", alert_type="warning")
+    #CONFIGS
+    baseline['QUARTERLY'] = 'Baseline: '+ baseline['QUARTERLY'].unique()[0] + '_' + baseline['QUARTERLY'].unique()[-1]
+    baseline['MONTHLY'] = 'Baseline: '+ baseline['MONTHLY'].unique()[0] + '_' + baseline['MONTHLY'].unique()[-1]
+    baseline['WEEKLY'] = 'Baseline: '+ baseline['WEEKLY'].unique()[0] + '_' + baseline['WEEKLY'].unique()[-1]
+    #AUC
+    auc_b = baseline.groupby([period_metrics.value]).apply(AUC)
+    auc_p = prod.groupby([period_metrics.value]).apply(AUC)
+    baseline_auc = pn.widgets.DataFrame(auc_b)
+    prod_auc = pn.widgets.DataFrame(auc_p,name = 'AUC') #autosize_mode='fit_columns'
     
-        # build the results column
-        results = pn.Column("##### 🎉 Here are the results!", img)
+    from sklearn.metrics import roc_curve
+    from holoviews import Slope
+    b_label = baseline.MONTHLY.min()
+    FPR,TPR,T = roc_curve(baseline['TARGET'],baseline['SCORE'])
+    roc_baseline = pd.concat([pd.Series(TPR), pd.Series(FPR)], keys = ['TPR', 'FPR'], axis = 1)
+    roc_baseline_p = roc_baseline.hvplot.line(x ='FPR', y = 'TPR', label = "Baseline", color = 'red')
+
+    roc_plot = prod.groupby([period_metrics.value]).apply(ROC).reset_index(level = 0).hvplot.line(x ='FPR', y = 'TPR', title = "%s ROC (Production VS %s)"%(period_metrics.value, b_label),
+                                                        groupby = period_metrics.value, width = 600, height = 500, label = "Prod",
+                                                        xlim = (0,1), ylim = (0,1), grid = True) *  Slope(slope=1, y_intercept=0).opts(color='black', line_dash='dashed') * roc_baseline_p 
+    #KS
+    ks_b = baseline.groupby([period_metrics.value]).apply(ks)
+    ks_p = prod.groupby([period_metrics.value]).apply(ks)
+    baseline_ks = pn.widgets.DataFrame(ks_b)
+    prod_ks = pn.widgets.DataFrame(ks_p,name = 'AUC') #autosize_mode='fit_columns'
+
+    #LIFT
+    baseline_lift_raw, baseline_lift_raw_bins = lift_init(df = baseline)
+    baseline_lift_raw = baseline_lift_raw.rename(columns = {'Baseline': b_label})
+    prod_lift_raw,  prod_lift_raw_bins = lift_init(df = prod, baseline = baseline, is_baseline = False)
+    cols_b = baseline_lift_raw.columns.drop(['BINS', 'SCORE'])
+    cols = prod_lift_raw.columns.drop(['BINS', 'SCORE'])
+
+    baseline_lift = baseline_lift_raw.loc[baseline_lift_raw.BINS =='10',cols_b]
+    prod_lift = prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols]
+    # prod_lift = pd.concat([prod_lift.dropna(subset = [col]).dropna(axis = 1).reset_index(drop = 1) for col in prod_lift][1:], axis = 1)
+    lift_table = prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols].melt(id_vars="SCORE_BAND", 
+                                var_name='column', 
+                                value_name='value').dropna().reset_index(drop = True).rename(columns = {'column':period_metrics.value , 'value': 'Target_PCT'})
+    # print(prod_lift_raw_bins.loc[prod_lift_raw_bins.BINS ==10])
+    lift_table = lift_table.hvplot.table(groupby = period_metrics.value, title="%s Lift Table"%period_metrics.value, hover = True, responsive=True, 
+                shared_axes= False, fit_columns = True,
+                padding=True , index_position = 0, fontscale = 1.5)
+    # print(prod_lift_raw.loc[prod_lift_raw.BINS =='10',cols])
+    # print(baseline_lift_raw.loc[baseline_lift_raw.BINS == '10',cols_b])
+    prod_lift_raw['BINS'] = prod_lift_raw['BINS'].astype(int)
+    baseline_lift_raw['BINS'] = baseline_lift_raw['BINS'].astype(int)
     
-        for class_item, class_likelihood in zip(class_items, class_likelihoods):
-            row_label = pn.widgets.StaticText(
-                name=class_item.strip(), value=f"{class_likelihood:.2%}", align="center"
-            )
-            row_bar = pn.indicators.Progress(
-                value=int(class_likelihood * 100),
-                sizing_mode="stretch_width",
-                bar_color="secondary",
-                margin=(0, 10),
-                design=pn.theme.Material,
-            )
-            results.append(pn.Column(row_label, row_bar))
-        yield results
-    finally:
-        main.disabled = False
-
-
-# create widgets
-randomize_url = pn.widgets.Button(name="Randomize URL", align="end")
-
-image_url = pn.widgets.TextInput(
-    name="Image URL to classify",
-    value=pn.bind(random_url, randomize_url),
-)
-class_names = pn.widgets.TextInput(
-    name="Comma separated class names",
-    placeholder="Enter possible class names, e.g. cat, dog",
-    value="cat, dog, parrot",
-)
+    prod_lift_raw_bins['SCORE_BAND'] = prod_lift_raw_bins['SCORE_BAND'].astype(str)
+    # prod_lift_raw_bins['BINS'] = prod_lift_raw_bins['BINS'].astype(str)
 
-input_widgets = pn.Column(
-    "##### 😊 Click randomize or paste a URL to start classifying!",
-    pn.Row(image_url, randomize_url),
-    class_names,
-)
+    baseline_lift_raw_bins['SCORE_BAND'] = baseline_lift_raw_bins['SCORE_BAND'].astype(str)
+    # baseline_lift_raw_bins['BINS'] = baseline_lift_raw_bins['BINS'].astype(str)  
 
-# add interactivity
-interactive_result = pn.panel(
-    pn.bind(process_inputs, image_url=image_url, class_names=class_names),
-    height=600,
-)
+    # print(prod_lift_raw.loc[:,list(cols)+['BINS']])
+    p1 = prod_lift_raw_bins.set_index('SCORE_BAND'
+                                                      ).reset_index().hvplot.line(x = 'SCORE_BAND', groupby = ['BINS', period_metrics.value],
+                                                          grid = True, width = 1200, height = 500,
+                                                          label = 'Production', rot = 45)
+    
+    # print(baseline_lift_raw_bins)  
+    # print(prod_lift_raw_bins)                                                    
+    p2 = prod_lift_raw_bins.set_index('SCORE_BAND'
+                                                        ).reset_index().hvplot.scatter(x = 'SCORE_BAND', groupby = ['BINS', period_metrics.value], grid = True, color='DarkBlue', label='Production', rot = 45) 
 
-# add footer
-footer_row = pn.Row(pn.Spacer(), align="center")
-for icon, url in ICON_URLS.items():
-    href_button = pn.widgets.Button(icon=icon, width=35, height=35)
-    href_button.js_on_click(code=f"window.open('{url}')")
-    footer_row.append(href_button)
-footer_row.append(pn.Spacer())
-
-# create dashboard
-main = pn.WidgetBox(
-    input_widgets,
-    interactive_result,
-    footer_row,
-)
+    b_label = baseline.MONTHLY.min()
+    # print(baseline_lift_raw.loc[baseline_lift_raw.BINS == '10',cols_b][b_label])
+    b1 = baseline_lift_raw_bins.hvplot.line(x = 'SCORE_BAND', groupby = ['BINS'],
+                                                            grid = True, width = 1200, height = 500,
+                                                            line_dash='dashed', color = 'black', label = b_label, rot = 45)
+
+    b2 = baseline_lift_raw_bins.hvplot.scatter(x = 'SCORE_BAND', groupby = ['BINS'], grid = True, color='DarkGreen', label = b_label, rot = 45) 
+
+    final_lift_plots = (p1*p2*b1*b2).opts(ylabel = '%target_rate_mean', title = "%s Lift Chart " % (period_metrics.value.title())) 
+
+    #LABEL_DRIFT
+    mean_score_prod = prod.groupby(period_metrics.value).agg(MEAN_SCORE=("SCORE","mean"), MEAN_TARGET=("TARGET","mean"),Count = ("TARGET","count"))
+    mean_score_base = baseline.groupby(period_metrics.value).agg(MEAN_SCORE=("SCORE","mean"), MEAN_TARGET=("TARGET","mean"),Count = ("TARGET","count"))
+    baseline_label_drift = pn.widgets.DataFrame(mean_score_base)
+    prod_label_drift = pn.widgets.DataFrame(mean_score_prod,name = 'DRIFT')
+
+    #Lift Tables
+    # gains_final_all,_ = gains_table_proba(prod,'TARGET', 'SCORE')
+    lift_data = pd.concat([baseline_lift, prod_lift], axis = 0)
+    lift_data = pd.concat([lift_data.dropna(subset = [col]).dropna(axis = 1).reset_index(drop = 1) for col in lift_data][1:], axis = 1).dropna(axis = 1, how = 'any')
+    lift_data = lift_data.loc[:,~lift_data.columns.duplicated()].set_index('SCORE_BAND')
+    if (lift_data.shape[1] > 4) | (lift_data.shape[0] > 10):
+        prod_lift = pn.pane.Markdown('### Please download the csv as the lift table will congest the screen')
+    else:
+        prod_lift = pn.widgets.DataFrame(lift_data,name = 'LIFT')
+    #GAINS_TABLE
+    gains_final_prod,_ = gains_table_proba(prod,'TARGET', 'SCORE')
+    gains_final_base,_ = gains_table_proba(baseline,'TARGET', 'SCORE')
+    gains_final_base.index.names = [b_label]
+    gains_final_p = pn.widgets.DataFrame(gains_final_prod.set_index(['low','high']),name = 'GAINS',)
+    gains_final_b = pn.widgets.DataFrame(gains_final_base.set_index(['low','high']),name = 'GAINS',)
+
+    ece, bin_probamean, bin_ymean, bin_id, bin_count, bin_edges = expected_calibration_error(prod.TARGET.values, prod.SCORE.values)
+    error = pd.DataFrame(np.array([bin_probamean, bin_ymean]).T,columns= ["SCORE_MEAN", "TARGET_MEAN"])
+    error_plot = error.hvplot.scatter(x ='SCORE_MEAN', y = 'TARGET_MEAN', width = 800, height = 500, label = "Bin (Score vs Target Mean)", title = 'Model Scores Calibration (--- Perfect Calibration)',
+                                                        xlim = (0,1), ylim = (0,1), grid = True, xlabel = 'Bins Mean of Scores', ylabel = 'Bins Mean of Target') *  Slope(slope=1, y_intercept=0,legend = 'Perfect Calibration').opts(color='black', line_dash='dashed')  
+    variable_ = pn.pane.Alert('''### FJ Day Count: \n%s 
+    '''%(datetime.datetime.now() - pd.Timestamp('2023-03-06 03:27')), alert_type="success")
+    return pn.Tabs(
+        ('Metrics', pn.Column(
+                    pn.Row(intiate, intiate2, intiate3, width = 1200),
+                    '# PSI',
+                    pn.Row(psi_results, save_csv(psi_, 'PSI')),
+                    '# AUC',
+                    pn.Row(prod_auc, baseline_auc, save_csv(pd.concat([auc_b, auc_p], axis = 0), 'AUC')),
+                    '# KS',
+                    pn.Row(prod_ks, baseline_ks, save_csv(pd.concat([ks_b, ks_p], axis = 0), 'KS')),
+                    '# LABEL DRIFT',
+                    pn.Row(prod_label_drift, baseline_label_drift, save_csv(pd.concat([mean_score_base, mean_score_prod], axis = 0), 'LABEL_DRIFT')),
+                    '# LIFT TABLES',
+                    pn.Row(prod_lift, save_csv(lift_data, 'LIFT_TABLES')),
+                    '# GAINS TABLE',
+                    pn.Row(gains_final_b, gains_final_p, save_csv(pd.concat([gains_final_base, gains_final_prod], axis = 1), 'GAINS_TABLES')),
+                    get_xlsx(psi_, pd.concat([auc_b, auc_p], axis = 0), pd.concat([ks_b, ks_p], axis = 0), pd.concat([mean_score_base, mean_score_prod], axis = 0), lift_data, pd.concat([gains_final_base, gains_final_prod], axis = 1)), 
+                    pn.Row(variable_, width = 200),
+                             )
+        ), #sizing_mode='stretch_width'
+        ('Charts', pn.Column(pn.Row(roc_plot.opts(legend_position = 'bottom_right'), error_plot.opts(legend_position = 'top_left')) ,
+                             lift_table,
+                             final_lift_plots.opts(legend_position = 'bottom_right')
+                            )
+        )
+    
+    )
+        
+
+    # return pn.Tabs(
+    #         ('Analysis', pn.Column(
+    #             pn.Row(vol_ret, pn.layout.Spacer(width=20), pn.Column(div, table), sizing_mode='stretch_width'),
+    #             pn.Column(pn.Row(year, investment), return_curve, sizing_mode='stretch_width'),
+    #             sizing_mode='stretch_width')),
+    #         ('Timeseries', timeseries),
+    #         ('Log Return', pn.Column(
+    #             '## Daily normalized log returns',
+    #             'Width of distribution indicates volatility and center of distribution the mean daily return.',
+    #             log_ret_hists,
+    #             sizing_mode='stretch_width'
+    #         ))
+    #     )
+
+pn.Row(pn.Column(widgets), pn.layout.Spacer(width=30), run).servable()
+
+
+
+
+
+
+
+
+
+
+
+
+
+# Caveats
+# The maximum sizes set in either Bokeh or Tornado refer to the maximum size of the message that 
+# is transferred through the web socket connection, which is going to be larger than the actual 
+# size of the uploaded file since the file content is encoded in a base64 string. So if you set a
+# maximum size of 100 MB for your application, you should indicate your users that the upload
+# limit is a value that is less than 100 MB.
+
+# When a file whose size is larger than the limits is selected by a user, their browser/tab may
+# just crash. Alternatively the web socket connection can close (sometimes with an error message
+# printed in the browser console such as [bokeh] Lost websocket 0 connection, 1009 (message too 
+# big)) which means the application will become unresponsive and needs to be refreshed.
+
+# app = ...
+
+# MAX_SIZE_MB = 150
 
-title = "Panel Demo - Image Classification"
-pn.template.BootstrapTemplate(
-    title=title,
-    main=main,
-    main_max_width="min(50%, 698px)",
-    header_background="#F08080",
-).servable(title=title)
+# pn.serve(
+#     app,
+#     # Increase the maximum websocket message size allowed by Bokeh
+#     websocket_max_message_size=MAX_SIZE_MB*1024*1014,
+#     # Increase the maximum buffer size allowed by Tornado
+#     http_server_kwargs={'max_buffer_size': MAX_SIZE_MB*1024*1014}
+# )