app_funcs.py

import numpy as np
import matplotlib.pyplot as plt
import xgboost as xgb
import pickle
import os
import pandas as pd

# EDA
def check_dates(df, end_date):
    """
    Checks that the dataframe is in correct order
    """
    months_31 = {"01", "03", "05", "07", "08", "10", "12"}
    months_30 = {"04", "06", "09", "11"}
    months_28 = {"02"}
    for idx, row in df.iterrows():
        if(row["month_str"] == end_date):
            continue
        if(row["month_str"][5:7] in months_31):
            if (row["interval"] != np.timedelta64(31, "D")):
                return False
        if(row["month_str"][5:7] in months_30):
            if (row["interval"] != np.timedelta64(30, "D")):
                return False
        if(row["month_str"][5:7] in months_28):
            if (row["interval"] != np.timedelta64(28, "D") and int(row["month_str"][:4]) % 4 != 0):
                return False
            # Leap Year
            if (row["interval"] != np.timedelta64(29, "D") and int(row["month_str"][:4]) % 4 == 0):
                return False

    return True


# EDA
def plot_cohort(df, cohort_first_month, product):
    """
    Plots the specified cohort given product and the date of the cohort
    """
    df_ = get_sequence(df, cohort_first_month, product)
    x = np.array([x for x in range(df_.shape[0])])
    fig = plt.figure()
    ax = fig.gca()
    plt.plot(x, df_["percentage"])
    plt.grid()
    ax.set_xlim([0, df_.shape[0]])
    ax.set_ylim([0, 1])
    ax.set_xlabel("Months")
    ax.set_ylabel("Percentage")
    ttle = f"{cohort_first_month} | {product}"
    ax.set_title(ttle)
    plt.show()

##########################################################################################

# train
def plot_feature_importance(model, feature_names):
    """
    Plots the importance of the features of a XGB model
    """
    importances = model.feature_importances_
    indices = np.argsort(importances)[::-1]
    names = [feature_names[i] for i in indices]
    plt.figure(figsize=(10, 6))
    plt.title("Feature Importance")
    plt.bar(range(len(importances)), importances[indices])
    plt.xticks(range(len(importances)), names, rotation=90)
    plt.show()

##########################################################################################

# evaluate
def get_sequence(df, cohort_first_month, product):
    """
    Gets the dataframe of a sequence given the product and the date of the cohort
    """
    df_ = df[df["cohort_first_month"] == cohort_first_month]
    df_ = df_[df_["cohort_first_product"] == product]
    return df_



# evaluate
def plot_true_and_predicted(y_true, y_pred, cohort, product):
    """
    Plots the true and predicted time-series given a cohort and a product
    Every step is of the predicted is given the true t-1 datapoint. Its does not
    create an entire sequence from predictions. 
    """
    x = np.array([x for x in range(y_true.shape[0])])
    fig = plt.figure()
    ax = fig.gca()
    plt.plot(x, y_true, label="Y True")
    plt.plot(x, y_pred, label="Y Pred")
    plt.grid()
    ax.set_xlim([0, y_true.shape[0]])
    ax.set_ylim([0, 1])
    ax.set_xlabel("Months")
    ax.set_ylabel("Percentage")
    ttle = f"{cohort} | {product}"
    ax.set_title(ttle)
    ax.legend()
    plt.show()


# evaluate
def get_product_one_hot_encode(product):
    """
    Gets a one hot encoded dataframe of the possible products for a row
    """
    products = {"1m":0,"3m":0,"4m":0}
    columns = ["product_1m", "product_3m", "product_4m"]
    products[product] = 1
    df = pd.DataFrame([products])
    df = df.rename(columns = {"1m": columns[0],
                              "3m": columns[1],
                              "4m": columns[2]})
    # print(df)
    return df
    
# evaluate
def get_month_one_hot_encode(month):
    """
    Gets a one hot encoded dataframe of the months
    """
    months = [0 for x in range(12)]
    columns = [f"month_{x}" for x in range(1,13)]
    months[month-1] = 1
    df = pd.DataFrame([months], columns=columns)
    # print(df)
    return df
    
# evaluate
def generate_new_data(df, date, cohort, product, model, columns_to_drop, n_points):
    """
    This function generates data for a cohort of a product, from a specified date. 
    It will use the predicion model, to generate the n consequent time steps of a cohort. 
    The datapoints will be generated given the previously generated datapoints, in an iterative 
    fashion
    """
    df_ = df[df["cohort_first_month"] == cohort]
    df_ = df_[df_["cohort_first_product"] == product]
    df_ = df_[df_["month"] == date]
    current_month = int(date[5:7])
    current_msa = df_["months_since_acquisition"].values[0]
    df_ = df_.drop(columns=columns_to_drop)
    columns = df_.columns
    product_ohe = get_product_one_hot_encode(product)
    datapoint = df_.copy()
    counter = 0
    while(counter < n_points):
        prediction = model.predict(datapoint)
        # print(prediction)
        current_month = (current_month%12)+1
        month_ohe = get_month_one_hot_encode(current_month)
        current_msa += 1
        new_row = pd.DataFrame([current_msa], columns=[columns[0]])
        new_row[columns[1]] = prediction[0]
        new_row = new_row.join(product_ohe)
        new_row = new_row.join(month_ohe)
        df_ = pd.concat([df_,new_row], ignore_index=True)

        datapoint = new_row.copy()
        counter +=1

    return df_

# evaluate
def plot_example_from_case(historical, predicted, x_lim, product):
    """
    With the generated data, it plots the historical true data, and in a dotted line
    the data that was predicted by the model for the subsequent datapoints. 
    """
    x_historical = np.array([x for x in range(historical.shape[0])])
    x_predicted= np.array([x + historical.shape[0]-1 for x in range(predicted.shape[0])])
    y_historical = historical["percentage"]
    y_predicted = predicted["percentage"]
    cohort_date = historical.iloc[0]["cohort_first_month"].strftime('%Y-%m-%d')
    fig = plt.figure()
    ax = fig.gca()
    plt.plot(x_historical, y_historical, label="historical", color="blue", linestyle="-")
    plt.plot(x_predicted, y_predicted, label="predicted", color="blue", linestyle="--")
    plt.grid()
    ax.set_xlim([0, x_lim])
    ax.set_ylim([0, 1])
    ax.set_xlabel("Months")
    ax.set_ylabel("Percentage")
    ttle = f" Cohort {cohort_date} | Product {product}"
    ax.set_title(ttle)
    ax.legend()
    plt.show()

    return fig