Spaces:
Sleeping
Sleeping
| """ | |
| Ashley Wu | |
| Generates insights into the data given | |
| """ | |
| import pandas as pd | |
| import gradio as gr | |
| import utils | |
| def calculate_stats(df): | |
| """ | |
| Create a dataframe with each column's mean, median, max, min, quartiles for numeric columns and unique values, value counts, and mode for categorical columns as well as missing values | |
| Args: | |
| dataframe: The pandas DataFrame to analyze | |
| Returns: | |
| A DataFrame with column statistics | |
| """ | |
| df = utils.detect_categories(df) | |
| df = utils.detect_dates(df) | |
| if df is None or len(df) == 0: | |
| return None, None | |
| numeric_data = pd.DataFrame() | |
| categorical_data = pd.DataFrame() | |
| for column in df.columns: | |
| dtype = df[column].dtype | |
| if dtype.kind == 'i' or dtype.kind == 'f' or dtype.kind == 'u' or dtype.kind == 'c': | |
| numeric_data[column] = df[column] | |
| elif dtype.name == 'category': | |
| categorical_data[column] = df[column] | |
| elif dtype.kind == 'M' or dtype.kind == 'm': | |
| continue | |
| else: | |
| gr.Warning(f"dType of {column} could not be recognised!", duration=5) | |
| numeric_data = calcualte_numeric_data(numeric_data) | |
| categorical_data = calculate_categorical_data(categorical_data) | |
| return numeric_data, categorical_data | |
| def calculate_categorical_data(df): | |
| """ | |
| Calculates data for categorical data exploration | |
| Args: | |
| df: dataframe | |
| Returns | |
| table: dataframe | |
| """ | |
| table = pd.DataFrame({ | |
| 'Column': pd.Series(dtype='string'), | |
| 'Mode': pd.Series(dtype='string'), | |
| 'Missing Values': pd.Series(dtype='int16'), | |
| 'Value Counts': pd.Series(dtype='string'), | |
| }) | |
| for column in df.columns: | |
| mode = df[column].mode()[0] | |
| count = str(df[column].value_counts().to_dict()) | |
| missing_values = df[column].isnull().sum() | |
| table.loc[len(table)] = [column, mode, missing_values, count] | |
| return table | |
| def calcualte_numeric_data(df): | |
| """ | |
| Calcualtes statistics for numerical data | |
| Args: | |
| df: dataframe | |
| Returns: | |
| table: df with column statistics | |
| """ | |
| table = pd.DataFrame({ | |
| 'Column': pd.Series(dtype='string'), | |
| 'Mean': pd.Series(dtype='float16'), | |
| 'Median': pd.Series(dtype='float16'), | |
| 'Max': pd.Series(dtype='float16'), | |
| 'Min': pd.Series(dtype='float16'), | |
| 'Bottom Quartile': pd.Series(dtype='float16'), | |
| 'Top Quartile': pd.Series(dtype='float16'), | |
| 'Missing Values': pd.Series(dtype='int16'), | |
| }) | |
| for column in df.columns: | |
| mean = df[column].mean() | |
| median = df[column].median() | |
| max_val = df[column].max() | |
| min_val = df[column].min() | |
| q25 = df[column].quantile(.25) | |
| q75 = df[column].quantile(.75) | |
| missing_values = df[column].isnull().sum() | |
| table.loc[len(table)] = [column, mean, median, max_val, min_val, q25, q75, missing_values] | |
| return table | |
| def correlation_matrix(df): | |
| """ | |
| Create a correleation matrix for the given dataframe | |
| Args: | |
| df: dataframe | |
| Returns: | |
| correlation_matrix: numpy array | |
| """ | |
| for col in df.columns: | |
| if df[col].dtype.name == 'object': | |
| df = df.drop(col, axis=1) | |
| return df.corr() | |
| def max_min_df(df): | |
| """ | |
| Find the max and min of a data frame | |
| Arg: | |
| df: dataframe | |
| Returns: | |
| max_min_df: dataframe | |
| """ | |
| for col in df.columns: | |
| if df[col].dtype.name == 'object': | |
| df = df.drop(col, axis=1) | |
| df_max = df.max() | |
| df_min = df.min() | |
| result = pd.DataFrame({ | |
| 'Column': df_max.index, | |
| 'Max': df_max.values, | |
| 'Min': df_min.values | |
| }) | |
| return result | |
| def std(df): | |
| """ | |
| Finds the standard deviation of the dataframe | |
| Args: | |
| df: dataframe | |
| Returns: | |
| std: dataframe | |
| """ | |
| for col in df.columns: | |
| if df[col].dtype.name == 'object': | |
| df = df.drop(col, axis=1) | |
| std_series = df.std() | |
| result = std_series.to_frame(name='Standard Deviation').reset_index() | |
| result.columns = ['Column', 'Standard Deviation'] | |
| return result | |
| def var(df): | |
| """ | |
| Finds the variance of values in a dataframe | |
| Args: | |
| df: dataframe | |
| Returns: | |
| var: df | |
| """ | |
| for col in df.columns: | |
| if df[col].dtype.name == 'object': | |
| df = df.drop(col, axis=1) | |
| var_series = df.var() | |
| result = var_series.to_frame(name='Variance').reset_index() | |
| result.columns = ['Column', 'Variance'] | |
| return result |