""" Ashley Wu Generates insights into the data given """ import pandas as pd import gradio as gr import utils def calculate_stats(df): """ Create a dataframe with each column's mean, median, max, min, quartiles for numeric columns and unique values, value counts, and mode for categorical columns as well as missing values Args: dataframe: The pandas DataFrame to analyze Returns: A DataFrame with column statistics """ df = utils.detect_categories(df) df = utils.detect_dates(df) if df is None or len(df) == 0: return None, None numeric_data = pd.DataFrame() categorical_data = pd.DataFrame() for column in df.columns: dtype = df[column].dtype if dtype.kind == 'i' or dtype.kind == 'f' or dtype.kind == 'u' or dtype.kind == 'c': numeric_data[column] = df[column] elif dtype.name == 'category': categorical_data[column] = df[column] elif dtype.kind == 'M' or dtype.kind == 'm': continue else: gr.Warning(f"dType of {column} could not be recognised!", duration=5) numeric_data = calcualte_numeric_data(numeric_data) categorical_data = calculate_categorical_data(categorical_data) return numeric_data, categorical_data def calculate_categorical_data(df): """ Calculates data for categorical data exploration Args: df: dataframe Returns table: dataframe """ table = pd.DataFrame({ 'Column': pd.Series(dtype='string'), 'Mode': pd.Series(dtype='string'), 'Missing Values': pd.Series(dtype='int16'), 'Value Counts': pd.Series(dtype='string'), }) for column in df.columns: mode = df[column].mode()[0] count = str(df[column].value_counts().to_dict()) missing_values = df[column].isnull().sum() table.loc[len(table)] = [column, mode, missing_values, count] return table def calcualte_numeric_data(df): """ Calcualtes statistics for numerical data Args: df: dataframe Returns: table: df with column statistics """ table = pd.DataFrame({ 'Column': pd.Series(dtype='string'), 'Mean': pd.Series(dtype='float16'), 'Median': pd.Series(dtype='float16'), 'Max': pd.Series(dtype='float16'), 'Min': pd.Series(dtype='float16'), 'Bottom Quartile': pd.Series(dtype='float16'), 'Top Quartile': pd.Series(dtype='float16'), 'Missing Values': pd.Series(dtype='int16'), }) for column in df.columns: mean = df[column].mean() median = df[column].median() max_val = df[column].max() min_val = df[column].min() q25 = df[column].quantile(.25) q75 = df[column].quantile(.75) missing_values = df[column].isnull().sum() table.loc[len(table)] = [column, mean, median, max_val, min_val, q25, q75, missing_values] return table def correlation_matrix(df): """ Create a correleation matrix for the given dataframe Args: df: dataframe Returns: correlation_matrix: numpy array """ for col in df.columns: if df[col].dtype.name == 'object': df = df.drop(col, axis=1) return df.corr() def max_min_df(df): """ Find the max and min of a data frame Arg: df: dataframe Returns: max_min_df: dataframe """ for col in df.columns: if df[col].dtype.name == 'object': df = df.drop(col, axis=1) df_max = df.max() df_min = df.min() result = pd.DataFrame({ 'Column': df_max.index, 'Max': df_max.values, 'Min': df_min.values }) return result def std(df): """ Finds the standard deviation of the dataframe Args: df: dataframe Returns: std: dataframe """ for col in df.columns: if df[col].dtype.name == 'object': df = df.drop(col, axis=1) std_series = df.std() result = std_series.to_frame(name='Standard Deviation').reset_index() result.columns = ['Column', 'Standard Deviation'] return result def var(df): """ Finds the variance of values in a dataframe Args: df: dataframe Returns: var: df """ for col in df.columns: if df[col].dtype.name == 'object': df = df.drop(col, axis=1) var_series = df.var() result = var_series.to_frame(name='Variance').reset_index() result.columns = ['Column', 'Variance'] return result