Spaces:
Sleeping
Sleeping
Upload 5 files
Browse files- src/AutoVisualizer/__init__.py +0 -0
- src/AutoVisualizer/categorical_viz.py +283 -0
- src/AutoVisualizer/numerical_viz.py +233 -0
- src/AutoVisualizer/processing.py +250 -0
- src/app.py +222 -0
src/AutoVisualizer/__init__.py
ADDED
|
File without changes
|
src/AutoVisualizer/categorical_viz.py
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Module that will handle plots like bar chart, count plot, pie chart, etc.
|
| 2 |
+
import plotly.express as px
|
| 3 |
+
import streamlit as st
|
| 4 |
+
import numpy as np
|
| 5 |
+
import pandas as pd
|
| 6 |
+
from plotly.subplots import make_subplots
|
| 7 |
+
import plotly.graph_objs as go
|
| 8 |
+
|
| 9 |
+
# Subplotting Function
|
| 10 |
+
def combine_figures_as_subplots(figures: list, rows_per_column: int = 2):
|
| 11 |
+
"""
|
| 12 |
+
Combines a list of Plotly figures into a single subplot layout.
|
| 13 |
+
|
| 14 |
+
Args:
|
| 15 |
+
figures (list): A list of individual Plotly figure objects to be arranged as subplots.
|
| 16 |
+
rows_per_column (int, optional): The number of subplot rows per column. Defaults to 2.
|
| 17 |
+
|
| 18 |
+
Returns:
|
| 19 |
+
plotly.graph_objects.Figure: A single Plotly figure containing all input figures as subplots.
|
| 20 |
+
"""
|
| 21 |
+
total = len(figures)
|
| 22 |
+
cols = 1 if total <= rows_per_column else 2
|
| 23 |
+
rows = (total + cols - 1) // cols # Ceiling division to compute total rows
|
| 24 |
+
|
| 25 |
+
# Determine subplot types for each cell
|
| 26 |
+
specs = []
|
| 27 |
+
for i in range(rows):
|
| 28 |
+
row_specs = []
|
| 29 |
+
for j in range(cols):
|
| 30 |
+
idx = i * cols + j
|
| 31 |
+
if idx < total and figures[idx].data and figures[idx].data[0].type == "pie":
|
| 32 |
+
row_specs.append({"type": "domain"})
|
| 33 |
+
else:
|
| 34 |
+
row_specs.append({"type": "xy"})
|
| 35 |
+
specs.append(row_specs)
|
| 36 |
+
|
| 37 |
+
# Create subplots with correct specs
|
| 38 |
+
subplot_fig = make_subplots(
|
| 39 |
+
rows=rows, cols=cols,
|
| 40 |
+
subplot_titles=[fig.layout.title.text for fig in figures],
|
| 41 |
+
specs=specs
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
# Add each figure's traces to the corresponding subplot cell
|
| 45 |
+
for idx, fig in enumerate(figures):
|
| 46 |
+
row = (idx // cols) + 1
|
| 47 |
+
col = (idx % cols) + 1
|
| 48 |
+
|
| 49 |
+
for trace in fig.data:
|
| 50 |
+
subplot_fig.add_trace(trace, row=row, col=col)
|
| 51 |
+
|
| 52 |
+
# Only update axes for non-pie charts
|
| 53 |
+
if fig.data[0].type != "pie":
|
| 54 |
+
xaxis_title = fig.layout.xaxis.title.text
|
| 55 |
+
yaxis_title = fig.layout.yaxis.title.text
|
| 56 |
+
subplot_fig.update_xaxes(title_text=xaxis_title, row=row, col=col)
|
| 57 |
+
subplot_fig.update_yaxes(title_text=yaxis_title, row=row, col=col)
|
| 58 |
+
|
| 59 |
+
# Final layout adjustments
|
| 60 |
+
subplot_fig.update_layout(height=350 * rows, showlegend=False)
|
| 61 |
+
return subplot_fig
|
| 62 |
+
|
| 63 |
+
# Count Plots Function
|
| 64 |
+
def generate_count_plots(df: pd.DataFrame, categorical_column: str, max_label_len: int = 10):
|
| 65 |
+
"""
|
| 66 |
+
Generates a count plot (bar chart) for a given categorical column in a DataFrame.
|
| 67 |
+
|
| 68 |
+
Args:
|
| 69 |
+
df (pd.DataFrame): The input dataset.
|
| 70 |
+
categorical_column (str): The name of the categorical column to plot.
|
| 71 |
+
max_label_len (int, optional): Maximum number of characters to display on x-axis labels.
|
| 72 |
+
Longer labels are truncated with an ellipsis. Defaults to 10.
|
| 73 |
+
|
| 74 |
+
Returns:
|
| 75 |
+
list: A list containing a single Plotly bar chart figure.
|
| 76 |
+
"""
|
| 77 |
+
plots = []
|
| 78 |
+
# Compute value counts for the selected column
|
| 79 |
+
value_counts = df[categorical_column].value_counts().reset_index()
|
| 80 |
+
value_counts.columns = [categorical_column, "Count"]
|
| 81 |
+
|
| 82 |
+
# Add truncated label column
|
| 83 |
+
value_counts["Truncated"] = value_counts[categorical_column].apply(
|
| 84 |
+
lambda x: x if len(str(x)) <= max_label_len else str(x)[:max_label_len] + "β¦"
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
# Create a bar plot
|
| 88 |
+
fig = px.bar(
|
| 89 |
+
value_counts, x="Truncated", y="Count", color="Truncated",
|
| 90 |
+
title=f"Count Plot for '{categorical_column}'",
|
| 91 |
+
labels={categorical_column: "Truncated", "Count": "Frequency"},
|
| 92 |
+
custom_data=[value_counts[categorical_column]]
|
| 93 |
+
)
|
| 94 |
+
|
| 95 |
+
# Use full label in hover template
|
| 96 |
+
fig.update_traces(
|
| 97 |
+
hovertemplate=f"{categorical_column}=%{{customdata}}<br>Count=%{{y}}<extra></extra>"
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
fig.update_layout(showlegend=False, xaxis_title=categorical_column)
|
| 101 |
+
plots.append(fig)
|
| 102 |
+
return plots
|
| 103 |
+
|
| 104 |
+
# Bar Plot Function
|
| 105 |
+
def generate_bar_plots(df: pd.DataFrame, x_col: str, y_columns: list, max_label_len: int = 10):
|
| 106 |
+
"""
|
| 107 |
+
Generates bar plots showing the average of continuous features grouped by a categorical column.
|
| 108 |
+
|
| 109 |
+
Args:
|
| 110 |
+
df (pd.DataFrame): The input dataset.
|
| 111 |
+
x_col (str): The categorical column to group by (x-axis).
|
| 112 |
+
y_columns (list): List of continuous columns to aggregate and plot (y-axis).
|
| 113 |
+
max_label_len (int, optional): Maximum number of characters for x-axis labels.
|
| 114 |
+
Labels longer than this are truncated. Defaults to 10.
|
| 115 |
+
|
| 116 |
+
Returns:
|
| 117 |
+
list: A list of Plotly bar chart figures.
|
| 118 |
+
"""
|
| 119 |
+
plots = []
|
| 120 |
+
|
| 121 |
+
for y_col in y_columns:
|
| 122 |
+
# Compute mean aggregation
|
| 123 |
+
agg_func = df.groupby(x_col)[y_col].mean().reset_index()
|
| 124 |
+
|
| 125 |
+
# Add truncated label for x-axis ticks
|
| 126 |
+
agg_func["Truncated"] = agg_func[x_col].apply(
|
| 127 |
+
lambda x: x if len(str(x)) <= max_label_len else str(x)[:max_label_len] + "β¦"
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
fig = px.bar(
|
| 131 |
+
agg_func, x="Truncated", y=y_col, color="Truncated", # color by original to retain legend info
|
| 132 |
+
title=f"{x_col} vs {y_col} (Average)",
|
| 133 |
+
labels={"Truncated": "", y_col: y_col},
|
| 134 |
+
custom_data=[agg_func[x_col]],
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
# Format hover values
|
| 138 |
+
if (df[y_col].dropna() % 1 == 0).all():
|
| 139 |
+
hover_format = ".0f"
|
| 140 |
+
else:
|
| 141 |
+
hover_format = ".2f"
|
| 142 |
+
|
| 143 |
+
# Show full x value on hover instead of truncated
|
| 144 |
+
fig.update_traces(
|
| 145 |
+
hovertemplate=f"{x_col}=%{{customdata[0]}}<br>Average {y_col}=%{{y:{hover_format}}}<extra></extra>",
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
fig.update_layout(showlegend=False, xaxis_title=x_col)
|
| 149 |
+
plots.append(fig)
|
| 150 |
+
|
| 151 |
+
return plots
|
| 152 |
+
|
| 153 |
+
# Grouped Bar Plot Function
|
| 154 |
+
def generate_grouped_bar_plots(df: pd.DataFrame, x_columns: list, y_columns: list, max_label_len: int = 10):
|
| 155 |
+
"""
|
| 156 |
+
Generates grouped bar plots showing the average of continuous features
|
| 157 |
+
grouped by a categorical feature and further separated by a binary hue column.
|
| 158 |
+
|
| 159 |
+
Args:
|
| 160 |
+
df (pd.DataFrame): The input dataset.
|
| 161 |
+
x_columns (list): Categorical columns to consider for x-axis and hue roles.
|
| 162 |
+
y_columns (list): Continuous columns to be averaged and plotted.
|
| 163 |
+
max_label_len (int, optional): Maximum length of x-axis labels before truncation. Defaults to 10.
|
| 164 |
+
|
| 165 |
+
Returns:
|
| 166 |
+
list: A list of Plotly grouped bar chart figures.
|
| 167 |
+
"""
|
| 168 |
+
plots = []
|
| 169 |
+
|
| 170 |
+
# Split categorical columns
|
| 171 |
+
hue_candidates = [col for col in x_columns if df[col].nunique() <= 2]
|
| 172 |
+
x_candidates = [col for col in x_columns if df[col].nunique() > 2 and df[col].nunique() <= 10]
|
| 173 |
+
|
| 174 |
+
for x_col in x_candidates:
|
| 175 |
+
for y_col in y_columns:
|
| 176 |
+
for hue_col in hue_candidates:
|
| 177 |
+
# Group and aggregate
|
| 178 |
+
agg_func = df.groupby([x_col, hue_col])[y_col].mean().reset_index()
|
| 179 |
+
|
| 180 |
+
# Add truncated label for x-axis ticks
|
| 181 |
+
agg_func["Truncated"] = agg_func[x_col].apply(
|
| 182 |
+
lambda x: x if len(str(x)) <= max_label_len else str(x)[:max_label_len] + "β¦"
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
fig = px.bar(
|
| 186 |
+
agg_func,
|
| 187 |
+
x="Truncated",
|
| 188 |
+
y=y_col,
|
| 189 |
+
color=hue_col,
|
| 190 |
+
barmode="group",
|
| 191 |
+
title=f"{x_col} vs {y_col} grouped by {hue_col}",
|
| 192 |
+
labels={x_col: x_col, y_col: f"Average {y_col}", hue_col: hue_col},
|
| 193 |
+
custom_data=[agg_func[x_col], agg_func[hue_col]]
|
| 194 |
+
)
|
| 195 |
+
|
| 196 |
+
# Formatting hover text
|
| 197 |
+
if (df[y_col].dropna() % 1 == 0).all():
|
| 198 |
+
hover_format = ".0f"
|
| 199 |
+
else:
|
| 200 |
+
hover_format = ".2f"
|
| 201 |
+
|
| 202 |
+
# Show full x value on hover instead of truncated
|
| 203 |
+
fig.update_traces(
|
| 204 |
+
hovertemplate=f"{x_col}=%{{customdata[0]}}<br>{hue_col}=%{{customdata[1]}}<br>Average {y_col}=%{{y:{hover_format}}}<extra></extra>",
|
| 205 |
+
)
|
| 206 |
+
fig.update_layout(showlegend=False, xaxis_title=x_col)
|
| 207 |
+
plots.append(fig)
|
| 208 |
+
|
| 209 |
+
return plots
|
| 210 |
+
|
| 211 |
+
# Pie Charts Function
|
| 212 |
+
def generate_pie_plots(df: pd.DataFrame, categorical_column: str, max_label_len: int = 10):
|
| 213 |
+
"""
|
| 214 |
+
Generates a pie chart showing the distribution of values for a given categorical column.
|
| 215 |
+
|
| 216 |
+
Args:
|
| 217 |
+
df (pd.DataFrame): The input dataset.
|
| 218 |
+
categorical_column (str): The categorical column for which to create a pie chart.
|
| 219 |
+
max_label_len (int, optional): Maximum length of labels (currently unused here but reserved for consistency). Defaults to 10.
|
| 220 |
+
|
| 221 |
+
Returns:
|
| 222 |
+
list: A list containing one Plotly pie chart figure.
|
| 223 |
+
"""
|
| 224 |
+
plots = []
|
| 225 |
+
# Count occurrences of each category
|
| 226 |
+
value_counts = df[categorical_column].value_counts().reset_index()
|
| 227 |
+
value_counts.columns = [categorical_column, "Count"]
|
| 228 |
+
|
| 229 |
+
fig = px.pie(
|
| 230 |
+
value_counts, names=categorical_column, values="Count",
|
| 231 |
+
hole=0.3,
|
| 232 |
+
title=f"Pie Plot for '{categorical_column}'",
|
| 233 |
+
)
|
| 234 |
+
# Use full label in hover template
|
| 235 |
+
fig.update_traces(
|
| 236 |
+
textinfo='value',
|
| 237 |
+
hovertemplate=f"{categorical_column}=%{{label}}<br>Percentage=%{{percent}}<extra></extra>"
|
| 238 |
+
)
|
| 239 |
+
|
| 240 |
+
fig.update_layout(showlegend=False)
|
| 241 |
+
plots.append(fig)
|
| 242 |
+
return plots
|
| 243 |
+
|
| 244 |
+
# HeatMap Function
|
| 245 |
+
# Heatmap showing correlated categorical features based on a numerical target column
|
| 246 |
+
def generate_categorical_correlation_heatmap(df: pd.DataFrame, numerical_col: str, categorical_columns: list):
|
| 247 |
+
"""
|
| 248 |
+
Generates a heatmap showing correlations between categorical features based on a numeric target column.
|
| 249 |
+
|
| 250 |
+
This function encodes each categorical feature by replacing its categories with the mean value
|
| 251 |
+
of the numeric column for that category. Then it computes the Pearson correlation between the
|
| 252 |
+
encoded categorical features.
|
| 253 |
+
|
| 254 |
+
Args:
|
| 255 |
+
df (pd.DataFrame): The input DataFrame.
|
| 256 |
+
numerical_col (str): The target numeric column to base encoding on.
|
| 257 |
+
categorical_columns (list): List of categorical feature names to analyze.
|
| 258 |
+
|
| 259 |
+
Returns:
|
| 260 |
+
list: A list containing one Plotly heatmap figure visualizing the correlation matrix.
|
| 261 |
+
"""
|
| 262 |
+
plots = []
|
| 263 |
+
encoded_df = pd.DataFrame()
|
| 264 |
+
|
| 265 |
+
# Encode categorical features using mean of numeric target per category
|
| 266 |
+
for col in categorical_columns:
|
| 267 |
+
temp = df[[col, numerical_col]].dropna()
|
| 268 |
+
means = temp.groupby(col)[numerical_col].mean()
|
| 269 |
+
encoded_df[col] = temp[col].map(means)
|
| 270 |
+
|
| 271 |
+
# Compute correlation matrix on the encoded values
|
| 272 |
+
corr_matrix = encoded_df.corr(method="pearson")
|
| 273 |
+
|
| 274 |
+
fig = px.imshow(
|
| 275 |
+
corr_matrix,
|
| 276 |
+
text_auto=True,
|
| 277 |
+
color_continuous_scale="RdBu",
|
| 278 |
+
aspect="auto",
|
| 279 |
+
title=f"Categorical Feature Correlation Heatmap Based on your Target : '{numerical_col}'",
|
| 280 |
+
)
|
| 281 |
+
|
| 282 |
+
plots.append(fig)
|
| 283 |
+
return plots
|
src/AutoVisualizer/numerical_viz.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Module that will handle plots like histogram, boxplot, density plot, etc.
|
| 2 |
+
import plotly.express as px
|
| 3 |
+
import streamlit as st
|
| 4 |
+
import numpy as np
|
| 5 |
+
import pandas as pd
|
| 6 |
+
import plotly.graph_objs as go
|
| 7 |
+
from scipy.stats import gaussian_kde
|
| 8 |
+
|
| 9 |
+
# Box Plot Function
|
| 10 |
+
def generate_box_plots(df: pd.DataFrame, x_col: str, y_columns: list, max_label_len: int = 10):
|
| 11 |
+
"""
|
| 12 |
+
Generates box plots for multiple numerical columns grouped by a categorical column.
|
| 13 |
+
|
| 14 |
+
This function creates a series of Plotly box plots to visualize the distribution,
|
| 15 |
+
spread, and potential outliers of each numerical column across the values of a given
|
| 16 |
+
categorical column. Long category labels are truncated for better readability.
|
| 17 |
+
|
| 18 |
+
Args:
|
| 19 |
+
df (pd.DataFrame): The input DataFrame.
|
| 20 |
+
x_col (str): Categorical column to group the box plots by.
|
| 21 |
+
y_columns (list): List of numerical columns to plot on the y-axis.
|
| 22 |
+
max_label_len (int, optional): Maximum character length for category labels on the x-axis (default is 10).
|
| 23 |
+
|
| 24 |
+
Returns:
|
| 25 |
+
list: A list of Plotly figure objects, each representing a box plot for a y_column grouped by x_col.
|
| 26 |
+
"""
|
| 27 |
+
plots = []
|
| 28 |
+
|
| 29 |
+
# Drop rows with missing values in relevant columns
|
| 30 |
+
df = df[[x_col] + y_columns].dropna()
|
| 31 |
+
|
| 32 |
+
# Truncate long category names
|
| 33 |
+
df["Truncated_X"] = df[x_col].apply(
|
| 34 |
+
lambda x: str(x) if len(str(x)) <= max_label_len else str(x)[:max_label_len] + "β¦"
|
| 35 |
+
)
|
| 36 |
+
|
| 37 |
+
for y_col in y_columns:
|
| 38 |
+
fig = px.box(
|
| 39 |
+
df,
|
| 40 |
+
x="Truncated_X", y=y_col,
|
| 41 |
+
color="Truncated_X",
|
| 42 |
+
points="outliers", # Show outliers
|
| 43 |
+
title=f"Box Plot of {y_col} by {x_col}",
|
| 44 |
+
labels={"Truncated_X": x_col, y_col: y_col},
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
fig.update_layout(showlegend=False)
|
| 48 |
+
plots.append(fig)
|
| 49 |
+
|
| 50 |
+
return plots
|
| 51 |
+
|
| 52 |
+
# HeatMap Functions
|
| 53 |
+
## Heatmap of numerical (continuous) features
|
| 54 |
+
def generate_numeric_correlation_heatmap(df: pd.DataFrame):
|
| 55 |
+
"""
|
| 56 |
+
Generates a heatmap to visualize Pearson correlation between numerical features.
|
| 57 |
+
|
| 58 |
+
This function computes the correlation matrix for all numeric columns in the
|
| 59 |
+
DataFrame and returns a heatmap figure representing the strength and direction
|
| 60 |
+
of linear relationships between features.
|
| 61 |
+
|
| 62 |
+
Args:
|
| 63 |
+
df (pd.DataFrame): The input DataFrame.
|
| 64 |
+
|
| 65 |
+
Returns:
|
| 66 |
+
list: A list containing a single Plotly heatmap figure if enough numeric
|
| 67 |
+
features exist; otherwise, returns an empty list.
|
| 68 |
+
"""
|
| 69 |
+
plots = []
|
| 70 |
+
|
| 71 |
+
# Filter numeric columns
|
| 72 |
+
numeric_df = df.select_dtypes(include=np.number)
|
| 73 |
+
|
| 74 |
+
if numeric_df.shape[1] < 2:
|
| 75 |
+
return plots # Not enough numeric features to compute correlation
|
| 76 |
+
|
| 77 |
+
corr_matrix = numeric_df.corr(method="pearson")
|
| 78 |
+
|
| 79 |
+
fig = px.imshow(
|
| 80 |
+
corr_matrix,
|
| 81 |
+
text_auto=True,
|
| 82 |
+
color_continuous_scale="RdBu",
|
| 83 |
+
aspect="auto",
|
| 84 |
+
title="Numerical Feature Correlation Heatmap",
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
plots.append(fig)
|
| 88 |
+
return plots
|
| 89 |
+
|
| 90 |
+
# Scatter Plot Function
|
| 91 |
+
def generate_scatter_plots(df: pd.DataFrame, feature_pairs: list, color_by: str = None):
|
| 92 |
+
"""
|
| 93 |
+
Generates scatter plots for given pairs of numerical features.
|
| 94 |
+
|
| 95 |
+
Args:
|
| 96 |
+
df (pd.DataFrame): The input DataFrame containing the features.
|
| 97 |
+
feature_pairs (list): List of tuples containing (x, y) column names for each plot.
|
| 98 |
+
color_by (str, optional): Column name to color points by. Must be in df. Defaults to None.
|
| 99 |
+
|
| 100 |
+
Returns:
|
| 101 |
+
list: A list of Plotly scatter plot figures, one for each feature pair.
|
| 102 |
+
"""
|
| 103 |
+
plots = []
|
| 104 |
+
|
| 105 |
+
for x_col, y_col in feature_pairs:
|
| 106 |
+
fig = px.scatter(
|
| 107 |
+
df.dropna(subset=[x_col, y_col]),
|
| 108 |
+
x=x_col,
|
| 109 |
+
y=y_col,
|
| 110 |
+
color=color_by if color_by in df.columns else None,
|
| 111 |
+
title=f"Scatter Plot: {x_col} vs {y_col}",
|
| 112 |
+
labels={x_col: x_col, y_col: y_col}
|
| 113 |
+
)
|
| 114 |
+
# Update title and legend if coloring is applied
|
| 115 |
+
if color_by:
|
| 116 |
+
fig.update_layout(title = f"Scatter Plot: {x_col} vs {y_col} color by {color_by}",showlegend=bool(color_by))
|
| 117 |
+
|
| 118 |
+
fig.update_layout(showlegend=bool(color_by))
|
| 119 |
+
plots.append(fig)
|
| 120 |
+
|
| 121 |
+
return plots
|
| 122 |
+
|
| 123 |
+
# Histogram Function
|
| 124 |
+
def generate_histograms(df: pd.DataFrame, numeric_columns: list, bins: int = 30):
|
| 125 |
+
"""
|
| 126 |
+
Generates histogram and KDE plots for each specified numeric column.
|
| 127 |
+
|
| 128 |
+
Args:
|
| 129 |
+
df (pd.DataFrame): The input DataFrame containing numeric features.
|
| 130 |
+
numeric_columns (list): List of column names to plot histograms for.
|
| 131 |
+
bins (int, optional): Number of bins to use in the histogram. Defaults to 30.
|
| 132 |
+
|
| 133 |
+
Returns:
|
| 134 |
+
list: A list of Plotly figures, each showing a histogram and KDE curve.
|
| 135 |
+
"""
|
| 136 |
+
plots = []
|
| 137 |
+
|
| 138 |
+
for col in numeric_columns:
|
| 139 |
+
data = df[col].dropna()
|
| 140 |
+
|
| 141 |
+
# Histogram
|
| 142 |
+
hist = go.Histogram(
|
| 143 |
+
x=data,
|
| 144 |
+
nbinsx=bins,
|
| 145 |
+
name='Histogram',
|
| 146 |
+
opacity=0.6
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
# KDE Curve
|
| 150 |
+
kde = gaussian_kde(data)
|
| 151 |
+
x_range = np.linspace(data.min(), data.max(), 200)
|
| 152 |
+
kde_curve = go.Scatter(
|
| 153 |
+
x=x_range,
|
| 154 |
+
y=kde(x_range) * len(data) * (data.max() - data.min()) / bins, # scaled to match histogram
|
| 155 |
+
name='KDE',
|
| 156 |
+
mode='lines',
|
| 157 |
+
line=dict(color='red')
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
# Combine into a single figure
|
| 161 |
+
fig = go.Figure(data=[hist, kde_curve])
|
| 162 |
+
fig.update_layout(
|
| 163 |
+
title=f"Histogram + KDE for '{col}'",
|
| 164 |
+
xaxis_title=col,
|
| 165 |
+
yaxis_title="Count",
|
| 166 |
+
barmode='overlay',
|
| 167 |
+
showlegend=True
|
| 168 |
+
)
|
| 169 |
+
|
| 170 |
+
plots.append(fig)
|
| 171 |
+
|
| 172 |
+
return plots
|
| 173 |
+
|
| 174 |
+
# Line Plots Function
|
| 175 |
+
def generate_line_plots(df: pd.DataFrame, date_component_cols: list, y_columns: list, freq: str = "M"):
|
| 176 |
+
"""
|
| 177 |
+
Generates line plots for given numerical columns based on reconstructed datetime columns.
|
| 178 |
+
|
| 179 |
+
This function identifies datetime-related components in the DataFrame (e.g., 'order_year', 'order_month', 'order_day'),
|
| 180 |
+
reconstructs them into actual datetime objects, and then plots the specified `y_columns` over time using the chosen frequency.
|
| 181 |
+
|
| 182 |
+
Args:
|
| 183 |
+
df (pd.DataFrame): The input DataFrame containing date components and numerical data.
|
| 184 |
+
date_component_cols (list): List of column names representing datetime parts (e.g., 'order_date_year', 'order_date_month').
|
| 185 |
+
y_columns (list): List of numeric columns to be plotted against the date.
|
| 186 |
+
freq (str, optional): Resampling frequency for datetime aggregation.
|
| 187 |
+
Options:
|
| 188 |
+
'D' = daily
|
| 189 |
+
'W' = weekly
|
| 190 |
+
'M' = monthly (default)
|
| 191 |
+
'Y' = yearly
|
| 192 |
+
|
| 193 |
+
Returns:
|
| 194 |
+
list: A list of Plotly line plot figures for each (prefix, y_column) pair.
|
| 195 |
+
"""
|
| 196 |
+
from collections import defaultdict
|
| 197 |
+
plots = []
|
| 198 |
+
|
| 199 |
+
# Group columns by prefix
|
| 200 |
+
groups = defaultdict(dict)
|
| 201 |
+
for col in date_component_cols:
|
| 202 |
+
for suffix in ['_year', '_month', '_day', '_weekday']:
|
| 203 |
+
if col.endswith(suffix):
|
| 204 |
+
prefix = col.replace(suffix, '')
|
| 205 |
+
groups[prefix][suffix] = col
|
| 206 |
+
|
| 207 |
+
# Reconstruct datetime and generate plots
|
| 208 |
+
for prefix, components in groups.items():
|
| 209 |
+
if all(k in components for k in ['_year', '_month', '_day']):
|
| 210 |
+
# Build a datetime column from year, month, day
|
| 211 |
+
temp_df = df[[components['_year'], components['_month'], components['_day']] + y_columns].dropna()
|
| 212 |
+
temp_df["__date__"] = pd.to_datetime({
|
| 213 |
+
'year': temp_df[components['_year']],
|
| 214 |
+
'month': temp_df[components['_month']],
|
| 215 |
+
'day': temp_df[components['_day']]
|
| 216 |
+
}, errors='coerce')
|
| 217 |
+
temp_df = temp_df.dropna(subset=["__date__"])
|
| 218 |
+
temp_df.set_index("__date__", inplace=True)
|
| 219 |
+
|
| 220 |
+
# Resample and plot
|
| 221 |
+
resampled_df = temp_df.resample(freq).mean().reset_index()
|
| 222 |
+
|
| 223 |
+
for col in y_columns:
|
| 224 |
+
fig = px.line(
|
| 225 |
+
resampled_df,
|
| 226 |
+
x="__date__",
|
| 227 |
+
y=col,
|
| 228 |
+
title=f"Line Plot: {prefix} vs {col}",
|
| 229 |
+
labels={"__date__": "Date", col: col},
|
| 230 |
+
)
|
| 231 |
+
plots.append(fig)
|
| 232 |
+
|
| 233 |
+
return plots
|
src/AutoVisualizer/processing.py
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Module that will handle processing tasks of the user dataset
|
| 2 |
+
import numpy as np
|
| 3 |
+
import pandas as pd
|
| 4 |
+
import streamlit as st
|
| 5 |
+
|
| 6 |
+
# A quick cleanliness checker function
|
| 7 |
+
def check_dataset_cleanliness(df):
|
| 8 |
+
issues_found = False
|
| 9 |
+
|
| 10 |
+
# 1. Columns with null values
|
| 11 |
+
null_cols = df.columns[df.isnull().any()].tolist()
|
| 12 |
+
if null_cols:
|
| 13 |
+
issues_found = True
|
| 14 |
+
st.warning(f"β οΈ These columns contain missing (NaN) values: {null_cols}")
|
| 15 |
+
|
| 16 |
+
# 2. Object columns that appear to be numeric (but aren't due to dirty values)
|
| 17 |
+
misclassified_numeric = []
|
| 18 |
+
for col in df.select_dtypes(include="object").columns:
|
| 19 |
+
non_null = df[col].dropna().astype(str)
|
| 20 |
+
sample_size = min(100, len(non_null))
|
| 21 |
+
if sample_size == 0:
|
| 22 |
+
continue # skip if column has no non-null values
|
| 23 |
+
sample = non_null.sample(sample_size, random_state=1)
|
| 24 |
+
numeric_like_ratio = sample.str.replace(",", "").str.replace(".", "", regex=False).str.isdigit().mean()
|
| 25 |
+
if numeric_like_ratio > 0.6:
|
| 26 |
+
misclassified_numeric.append(col)
|
| 27 |
+
|
| 28 |
+
if misclassified_numeric:
|
| 29 |
+
issues_found = True
|
| 30 |
+
st.warning(
|
| 31 |
+
f"β οΈ These columns are stored as `object` but mostly contain numeric values.\n"
|
| 32 |
+
f"This may be due to the presence of invalid or non-numeric entries in a few rows: {misclassified_numeric}"
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
# 3. Checking duplicate records
|
| 36 |
+
num_duplicates = df.duplicated().sum()
|
| 37 |
+
if num_duplicates > 0:
|
| 38 |
+
issues_found = True
|
| 39 |
+
st.warning(f"β οΈ Your dataset contains {num_duplicates} duplicated rows.")
|
| 40 |
+
|
| 41 |
+
# 4. Checking Constant Columns (No Variation)
|
| 42 |
+
constant_cols = [col for col in df.columns if df[col].nunique(dropna=False) <= 1]
|
| 43 |
+
if constant_cols:
|
| 44 |
+
issues_found = True
|
| 45 |
+
st.warning(f"β οΈ These columns contain only a single unique value and may be useless for analysis: {constant_cols}")
|
| 46 |
+
|
| 47 |
+
# 5. Suspiciously High Cardinality in Categorical Columns
|
| 48 |
+
high_card_cols = [col for col in df.select_dtypes(include='object') if df[col].nunique() > 100]
|
| 49 |
+
if high_card_cols:
|
| 50 |
+
issues_found = True
|
| 51 |
+
st.warning(f"β οΈ These object-type columns have unusually high unique values (possibly IDs or noisy data): {high_card_cols}")
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# Final message
|
| 55 |
+
if not issues_found:
|
| 56 |
+
st.success("β
No major issues detected. Dataset looks clean!")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# Function that will identify the task of the dataset
|
| 60 |
+
def task_type(df: pd.DataFrame, target_col: str) -> str:
|
| 61 |
+
"""
|
| 62 |
+
Determine the machine learning task type based on the target column.
|
| 63 |
+
|
| 64 |
+
Args:
|
| 65 |
+
df (pd.DataFrame): The input dataset.
|
| 66 |
+
target_col (str or None): The target column name, or None for unsupervised tasks.
|
| 67 |
+
|
| 68 |
+
Returns:
|
| 69 |
+
str: One of the following task types:
|
| 70 |
+
- "Classification" if the target is categorical.
|
| 71 |
+
- "Clustering" if the target is not provided.
|
| 72 |
+
- "Regression" if the target is numerical.
|
| 73 |
+
- "Unknown" if the type cannot be recognized.
|
| 74 |
+
"""
|
| 75 |
+
# Handle unsupervised case (no target)
|
| 76 |
+
if target_col == "No Target":
|
| 77 |
+
return "Clustering"
|
| 78 |
+
|
| 79 |
+
target_series = df[target_col]
|
| 80 |
+
dtype = target_series.dtype
|
| 81 |
+
n_unique = target_series.nunique()
|
| 82 |
+
|
| 83 |
+
# Check for classification
|
| 84 |
+
if dtype == 'object' or dtype == 'bool' or (dtype == 'category'):
|
| 85 |
+
return "Classification"
|
| 86 |
+
|
| 87 |
+
# Check for binary/multi-class classification represented as integers or floats
|
| 88 |
+
if dtype.kind in ['i', 'u', 'f']: # Integer types
|
| 89 |
+
if n_unique <= 10: # Arbitrary threshold for classification
|
| 90 |
+
return "Classification"
|
| 91 |
+
else:
|
| 92 |
+
return "Regression"
|
| 93 |
+
|
| 94 |
+
# Numeric types default to regression
|
| 95 |
+
if dtype.kind in ['i', 'u', 'f']:
|
| 96 |
+
return "Regression"
|
| 97 |
+
|
| 98 |
+
return "Unknown"
|
| 99 |
+
|
| 100 |
+
# Function that will identify if an object feature is truly categorical or not
|
| 101 |
+
def is_probably_categorical(series: pd.Series, threshold_unique: int = 50, threshold_ratio: float = 0.1) -> bool:
|
| 102 |
+
"""
|
| 103 |
+
Determines whether a given pandas Series is likely to be a categorical feature.
|
| 104 |
+
|
| 105 |
+
Args:
|
| 106 |
+
series (pd.Series): The input data column to analyze.
|
| 107 |
+
threshold_unique (int, optional (default=50)) : Maximum number of unique values for an object-type column to be considered categorical.
|
| 108 |
+
threshold_ratio (float, optional (default=0.1)) : Maximum ratio of unique values to total entries for object-type column to be treated as categorical.
|
| 109 |
+
|
| 110 |
+
Returns:
|
| 111 |
+
bool: True if the series is likely categorical, False otherwise.
|
| 112 |
+
"""
|
| 113 |
+
|
| 114 |
+
# Heuristic for object types (e.g., strings): avoid classifying high-cardinality fields as categorical
|
| 115 |
+
if series.dtype == 'object':
|
| 116 |
+
num_unique = series.nunique()
|
| 117 |
+
unique_ratio = num_unique / len(series)
|
| 118 |
+
|
| 119 |
+
if num_unique <= threshold_unique and unique_ratio <= threshold_ratio:
|
| 120 |
+
return True # categorical
|
| 121 |
+
else:
|
| 122 |
+
return False # high-cardinality non-categorical (like names)
|
| 123 |
+
|
| 124 |
+
# Explicit categorical or boolean data types are considered categorical
|
| 125 |
+
elif pd.api.types.is_categorical_dtype(series):
|
| 126 |
+
return True
|
| 127 |
+
elif pd.api.types.is_bool_dtype(series):
|
| 128 |
+
return True
|
| 129 |
+
|
| 130 |
+
return False
|
| 131 |
+
|
| 132 |
+
# Function that will identify if an numerical feature is discrete or not
|
| 133 |
+
def is_discrete(series: pd.Series, max_unique: int = 20) -> bool:
|
| 134 |
+
"""
|
| 135 |
+
Determine whether a numeric series should be considered discrete.
|
| 136 |
+
|
| 137 |
+
Args:
|
| 138 |
+
series (pd.Series): The input numeric data column to analyze.
|
| 139 |
+
max_unique (int, optional): Maximum number of unique values allowed
|
| 140 |
+
to treat a column as discrete. Default is 20.
|
| 141 |
+
|
| 142 |
+
Returns:
|
| 143 |
+
bool: True if the series is likely discrete, False otherwise.
|
| 144 |
+
"""
|
| 145 |
+
# Check if the series is of integer type
|
| 146 |
+
if pd.api.types.is_integer_dtype(series):
|
| 147 |
+
return series.nunique() <= max_unique
|
| 148 |
+
|
| 149 |
+
if pd.api.types.is_float_dtype(series):
|
| 150 |
+
# If all values are whole numbers AND unique count is low β treat as discrete
|
| 151 |
+
if series.dropna().apply(float.is_integer).all():
|
| 152 |
+
return series.nunique() <= max_unique
|
| 153 |
+
|
| 154 |
+
return False
|
| 155 |
+
|
| 156 |
+
# Function that will identify if an numerical feature is continuous or not
|
| 157 |
+
def is_continuous(series: pd.Series, max_unique: int = 20) -> bool:
|
| 158 |
+
"""
|
| 159 |
+
Determine whether a numeric series is continuous.
|
| 160 |
+
|
| 161 |
+
Args:
|
| 162 |
+
series (pd.Series): The input numeric data column to analyze.
|
| 163 |
+
max_unique (int, optional): Threshold for unique values. If a float-type column
|
| 164 |
+
contains only whole numbers and has fewer than this count, it is not considered continuous.
|
| 165 |
+
Default is 20.
|
| 166 |
+
|
| 167 |
+
Returns:
|
| 168 |
+
bool: True if the series is likely continuous, False otherwise.
|
| 169 |
+
"""
|
| 170 |
+
# Only float types are considered potentially continuous
|
| 171 |
+
if pd.api.types.is_float_dtype(series):
|
| 172 |
+
# If it's float but looks like discrete, then not continuous
|
| 173 |
+
all_whole_numbers = series.dropna().apply(float.is_integer).all()
|
| 174 |
+
if all_whole_numbers and series.nunique() <= max_unique:
|
| 175 |
+
return False
|
| 176 |
+
return True
|
| 177 |
+
return False
|
| 178 |
+
|
| 179 |
+
# Function that will identify if an feature is date-time format and then extract the time-based components
|
| 180 |
+
def parse_datetime_columns(df: pd.DataFrame) -> tuple[pd.DataFrame, list, list]:
|
| 181 |
+
"""
|
| 182 |
+
Detects and parses datetime columns in a DataFrame, and extracts useful
|
| 183 |
+
date and/or time components into new columns.
|
| 184 |
+
|
| 185 |
+
Args:
|
| 186 |
+
df (pd.DataFrame): Input dataset.
|
| 187 |
+
|
| 188 |
+
Returns:
|
| 189 |
+
tuple:
|
| 190 |
+
- pd.DataFrame: Updated DataFrame with extracted datetime components.
|
| 191 |
+
- list: List of original columns identified as datetime.
|
| 192 |
+
- list: List of newly extracted datetime-related feature names.
|
| 193 |
+
"""
|
| 194 |
+
datetime_cols = []
|
| 195 |
+
extracted_datetime = []
|
| 196 |
+
today = pd.Timestamp.today() # Just the date, no time
|
| 197 |
+
|
| 198 |
+
for col in df.columns:
|
| 199 |
+
if pd.api.types.is_datetime64_any_dtype(df[col]):
|
| 200 |
+
datetime_cols.append(col)
|
| 201 |
+
elif df[col].dtype == "object":
|
| 202 |
+
try:
|
| 203 |
+
converted = pd.to_datetime(df[col], errors="raise")
|
| 204 |
+
df[col] = converted
|
| 205 |
+
datetime_cols.append(col)
|
| 206 |
+
except Exception:
|
| 207 |
+
continue
|
| 208 |
+
|
| 209 |
+
for col in datetime_cols:
|
| 210 |
+
# Flags for what actually exists
|
| 211 |
+
has_date = True
|
| 212 |
+
has_time = True
|
| 213 |
+
|
| 214 |
+
# Check if all dates are "today" β probably not originally present
|
| 215 |
+
# if df[col].dt.normalize().nunique() == 1 and df[col].dt.normalize().iloc[0] == today:
|
| 216 |
+
if (df[col].dt.year == today.year).all() or (df[col].dt.month == today.month).all() or (df[col].dt.day == today.day).all():
|
| 217 |
+
has_date = False
|
| 218 |
+
|
| 219 |
+
# Check if all times are 00:00:00 β probably not originally present
|
| 220 |
+
if (df[col].dt.hour == 0).all() and (df[col].dt.minute == 0).all() and (df[col].dt.second == 0).all():
|
| 221 |
+
has_time = False
|
| 222 |
+
|
| 223 |
+
if has_date:
|
| 224 |
+
df[f"{col}_year"] = df[col].dt.year
|
| 225 |
+
df[f"{col}_month"] = df[col].dt.month
|
| 226 |
+
df[f"{col}_day"] = df[col].dt.day
|
| 227 |
+
df[f"{col}_weekday"] = df[col].dt.day_name()
|
| 228 |
+
extracted_datetime.extend([
|
| 229 |
+
f"{col}_year", f"{col}_month", f"{col}_day", f"{col}_weekday"
|
| 230 |
+
])
|
| 231 |
+
else:
|
| 232 |
+
df[f"{col}_year"] = np.nan
|
| 233 |
+
df[f"{col}_month"] = np.nan
|
| 234 |
+
df[f"{col}_day"] = np.nan
|
| 235 |
+
df[f"{col}_weekday"] = np.nan
|
| 236 |
+
|
| 237 |
+
if has_time:
|
| 238 |
+
df[f"{col}_hour"] = df[col].dt.hour
|
| 239 |
+
df[f"{col}_minute"] = df[col].dt.minute
|
| 240 |
+
extracted_datetime.extend([
|
| 241 |
+
f"{col}_hour", f"{col}_minute"
|
| 242 |
+
])
|
| 243 |
+
else:
|
| 244 |
+
df[f"{col}_hour"] = np.nan
|
| 245 |
+
df[f"{col}_minute"] = np.nan
|
| 246 |
+
|
| 247 |
+
# Remove any columns that are now entirely NaN
|
| 248 |
+
df = df.dropna(axis=1, how='all')
|
| 249 |
+
|
| 250 |
+
return df, datetime_cols, extracted_datetime
|
src/app.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import streamlit as st
|
| 2 |
+
import numpy as np
|
| 3 |
+
import pandas as pd
|
| 4 |
+
from AutoVisualizer.processing import check_dataset_cleanliness, task_type,is_probably_categorical, is_discrete, is_continuous, parse_datetime_columns
|
| 5 |
+
from AutoVisualizer.categorical_viz import combine_figures_as_subplots, generate_count_plots, generate_bar_plots, generate_grouped_bar_plots, generate_pie_plots, generate_categorical_correlation_heatmap
|
| 6 |
+
from AutoVisualizer.numerical_viz import generate_box_plots, generate_numeric_correlation_heatmap, generate_scatter_plots, generate_histograms, generate_line_plots
|
| 7 |
+
|
| 8 |
+
st.set_page_config(page_title= "Auto-Visualizer",page_icon= "π",layout="wide")
|
| 9 |
+
# col1, col2 = st.columns([0.3, 0.7])
|
| 10 |
+
|
| 11 |
+
with st.sidebar:
|
| 12 |
+
# Upload the dataset file (can upload only CSV, XLSX, JSON, XML)
|
| 13 |
+
uploaded_file = st.file_uploader("Upload your dataset file:", ["csv", "xlsx", "json", "xml"])
|
| 14 |
+
|
| 15 |
+
if uploaded_file is not None:
|
| 16 |
+
file_type = uploaded_file.name
|
| 17 |
+
|
| 18 |
+
try:
|
| 19 |
+
# Read the dataset through pandas
|
| 20 |
+
if file_type.endswith(".csv"):
|
| 21 |
+
df = pd.read_csv(uploaded_file)
|
| 22 |
+
elif file_type.endswith(".xlsx"):
|
| 23 |
+
df = pd.read_excel(uploaded_file)
|
| 24 |
+
elif file_type.endswith(".json"):
|
| 25 |
+
df = pd.read_json(uploaded_file)
|
| 26 |
+
else:
|
| 27 |
+
df = pd.read_xml(uploaded_file)
|
| 28 |
+
except Exception as e:
|
| 29 |
+
st.write("Error:", e)
|
| 30 |
+
|
| 31 |
+
with st.sidebar:
|
| 32 |
+
# Show a a Disclaimer to user to upload as clean data as possible
|
| 33 |
+
st.info("""
|
| 34 |
+
β οΈ **Heads up!** For the best experience, please upload a clean dataset.
|
| 35 |
+
|
| 36 |
+
This app is designed for *visualizing data*, not cleaning it.
|
| 37 |
+
|
| 38 |
+
π *Tip:* Use the quick checker below to spot potential issues.
|
| 39 |
+
""")
|
| 40 |
+
|
| 41 |
+
if st.button("Run Cleanliness Check"):
|
| 42 |
+
# Use session state to track button click
|
| 43 |
+
st.session_state.run_clean_check = True
|
| 44 |
+
st.divider()
|
| 45 |
+
|
| 46 |
+
# Display the cleanliness checker result on the main page (not sidebar) if button was clicked
|
| 47 |
+
if st.session_state.get("run_clean_check", False):
|
| 48 |
+
with st.expander("β‘οΈ See Cleanliness Checker Result"):
|
| 49 |
+
check_dataset_cleanliness(df)
|
| 50 |
+
|
| 51 |
+
# Display the user DataFrame
|
| 52 |
+
st.markdown("Your Dataset:")
|
| 53 |
+
st.dataframe(df, height=210)
|
| 54 |
+
st.divider()
|
| 55 |
+
|
| 56 |
+
# Extract column names from the dataset + add "No Target" element if there's no target in the user dataset
|
| 57 |
+
feature_list = list(df.columns)
|
| 58 |
+
target_selector = ["No Target"] + feature_list
|
| 59 |
+
|
| 60 |
+
with st.sidebar:
|
| 61 |
+
# Ask the user to select target column from their dataset
|
| 62 |
+
target_col = st.selectbox("Specify the target column in your dataset:", target_selector)
|
| 63 |
+
# Identify the task/type of dataset (i.e. classification/regression/clustering(if no target feature at all))
|
| 64 |
+
task = task_type(df, target_col)
|
| 65 |
+
# Display the task to user
|
| 66 |
+
st.write(f"π Task identified: **{task}**")
|
| 67 |
+
|
| 68 |
+
# Identify the date-time columns (if any) and extract new time-based components from it
|
| 69 |
+
# date_time_ls --> List that will store date-time feature names
|
| 70 |
+
# extracted_datetime --> List that will store extracted date-time feature names
|
| 71 |
+
df, date_time_ls, extracted_datetime = parse_datetime_columns(df)
|
| 72 |
+
|
| 73 |
+
# Remove date-time feature names as we already extracted time based components from it
|
| 74 |
+
feature_list = [x for x in feature_list if x not in date_time_ls]
|
| 75 |
+
|
| 76 |
+
categorical_ls = [] # List that will store categorical feature names
|
| 77 |
+
discrete_ls = [] # List that will store discrete feature names
|
| 78 |
+
continuous_ls = [] # List that will store continuous feature names
|
| 79 |
+
for feature in feature_list:
|
| 80 |
+
if is_probably_categorical(df[feature]):
|
| 81 |
+
categorical_ls.append(feature) # Calling Categorical Feature Identifier Function
|
| 82 |
+
elif is_discrete(df[feature]):
|
| 83 |
+
discrete_ls.append(feature) # Calling Discrete Feature Identifier Function
|
| 84 |
+
elif is_continuous(df[feature]):
|
| 85 |
+
continuous_ls.append(feature) # Calling Continuous Feature Identifier Function
|
| 86 |
+
# Add time based components that appear to be categorical
|
| 87 |
+
for feature in extracted_datetime:
|
| 88 |
+
if is_probably_categorical(df[feature]):
|
| 89 |
+
categorical_ls.append(feature)
|
| 90 |
+
|
| 91 |
+
# Creating Dialog Box to show identified features
|
| 92 |
+
@st.dialog("Identified/Extracted Features from your Dataset:-")
|
| 93 |
+
def open_dialog():
|
| 94 |
+
if categorical_ls:
|
| 95 |
+
with st.popover("Categorical Features", use_container_width= True):
|
| 96 |
+
st.code("\n".join([f"β’ {item}" for item in categorical_ls]))
|
| 97 |
+
if discrete_ls:
|
| 98 |
+
with st.popover("Discrete Features", use_container_width= True):
|
| 99 |
+
st.code("\n".join([f"β’ {item}" for item in discrete_ls]))
|
| 100 |
+
if continuous_ls:
|
| 101 |
+
with st.popover("Continuous Features", use_container_width= True):
|
| 102 |
+
st.code("\n".join([f"β’ {item}" for item in continuous_ls]))
|
| 103 |
+
if date_time_ls:
|
| 104 |
+
with st.popover("Date-Time Features", use_container_width= True):
|
| 105 |
+
st.code("\n".join([f"β’ {item}" for item in date_time_ls]))
|
| 106 |
+
with st.popover("Extracted features from your Date-Time like features", use_container_width= True):
|
| 107 |
+
st.code("\n".join([f"β’ {item}" for item in extracted_datetime]))
|
| 108 |
+
with st.sidebar:
|
| 109 |
+
# Calling the dialog box through a button
|
| 110 |
+
if st.button("See Your Feature Details"):
|
| 111 |
+
open_dialog()
|
| 112 |
+
|
| 113 |
+
# Generate the Plots
|
| 114 |
+
with st.spinner("Generating Plots.....", show_time= True):
|
| 115 |
+
if categorical_ls:
|
| 116 |
+
st.header("π Categorical Plots")
|
| 117 |
+
# 1. Count Plots
|
| 118 |
+
count_plots = []
|
| 119 |
+
for x_col in categorical_ls:
|
| 120 |
+
if df[x_col].nunique() <= 20:
|
| 121 |
+
count_plots.extend(generate_count_plots(df, x_col))
|
| 122 |
+
|
| 123 |
+
if count_plots:
|
| 124 |
+
st.subheader("Count Plots :-")
|
| 125 |
+
st.plotly_chart(combine_figures_as_subplots(count_plots), use_container_width=True)
|
| 126 |
+
|
| 127 |
+
# 2. Bar Plots
|
| 128 |
+
bar_plots = []
|
| 129 |
+
# (Categorical vs Discrete + Continuous)
|
| 130 |
+
for x_col in categorical_ls:
|
| 131 |
+
if df[x_col].nunique() <= 20:
|
| 132 |
+
bar_plots.extend(generate_bar_plots(df, x_col, discrete_ls + continuous_ls))
|
| 133 |
+
|
| 134 |
+
if bar_plots:
|
| 135 |
+
st.subheader("Bar Plots :-")
|
| 136 |
+
st.plotly_chart(combine_figures_as_subplots(bar_plots), use_container_width=True)
|
| 137 |
+
|
| 138 |
+
# 3. Grouped Bar Plots
|
| 139 |
+
grp_bar_plots = []
|
| 140 |
+
# (Categorical vs Discrete + Continuous)
|
| 141 |
+
grp_bar_plots.extend(generate_grouped_bar_plots(df, categorical_ls, discrete_ls + continuous_ls))
|
| 142 |
+
|
| 143 |
+
if grp_bar_plots:
|
| 144 |
+
st.subheader("Grouped Bar Plots :-")
|
| 145 |
+
st.plotly_chart(combine_figures_as_subplots(grp_bar_plots), use_container_width=True)
|
| 146 |
+
|
| 147 |
+
# 4. Pie Charts
|
| 148 |
+
pie_plots = []
|
| 149 |
+
for x_col in categorical_ls:
|
| 150 |
+
if df[x_col].nunique() <= 20:
|
| 151 |
+
pie_plots.extend(generate_pie_plots(df, x_col))
|
| 152 |
+
|
| 153 |
+
if pie_plots:
|
| 154 |
+
st.subheader("Pie Charts :-")
|
| 155 |
+
st.plotly_chart(combine_figures_as_subplots(pie_plots), use_container_width=True)
|
| 156 |
+
|
| 157 |
+
if continuous_ls:
|
| 158 |
+
st.header("π Numerical Plots")
|
| 159 |
+
# 5. Box Plots
|
| 160 |
+
box_plots = []
|
| 161 |
+
for x_col in categorical_ls:
|
| 162 |
+
if df[x_col].nunique() <= 10:
|
| 163 |
+
box_plots.extend(generate_box_plots(df, x_col, continuous_ls))
|
| 164 |
+
|
| 165 |
+
if box_plots:
|
| 166 |
+
st.subheader("Box Plots :-")
|
| 167 |
+
st.plotly_chart(combine_figures_as_subplots(box_plots), use_container_width=True)
|
| 168 |
+
|
| 169 |
+
# 6. Heat Maps
|
| 170 |
+
heat_maps = []
|
| 171 |
+
if task == 'Regression':
|
| 172 |
+
if categorical_ls:
|
| 173 |
+
heat_maps.extend(generate_categorical_correlation_heatmap(df, target_col, categorical_ls))
|
| 174 |
+
heat_maps.extend(generate_numeric_correlation_heatmap(df[continuous_ls]))
|
| 175 |
+
if heat_maps:
|
| 176 |
+
st.subheader("Heat Maps :-")
|
| 177 |
+
st.plotly_chart(combine_figures_as_subplots(heat_maps), use_container_width=True)
|
| 178 |
+
|
| 179 |
+
# 7. Scatter Plots
|
| 180 |
+
if len(continuous_ls) >= 2:
|
| 181 |
+
st.subheader("Scatter Plots")
|
| 182 |
+
scatter_plots = []
|
| 183 |
+
# Creation of unique feature pairs (no repetition like (B, A) if (A, B) is already used)
|
| 184 |
+
feature_pairs = []
|
| 185 |
+
for i in range(len(continuous_ls)):
|
| 186 |
+
for j in range(i + 1, len(continuous_ls)):
|
| 187 |
+
feature_pairs.append((continuous_ls[i], continuous_ls[j]))
|
| 188 |
+
selection = st.pills("Highlight using a categorical feature :- ", categorical_ls)
|
| 189 |
+
scatter_plots.extend(generate_scatter_plots(df, feature_pairs, selection))
|
| 190 |
+
if scatter_plots:
|
| 191 |
+
st.plotly_chart(combine_figures_as_subplots(scatter_plots), use_container_width=True)
|
| 192 |
+
|
| 193 |
+
# 8. Histograms
|
| 194 |
+
histograms = []
|
| 195 |
+
histograms.extend(generate_histograms(df, continuous_ls))
|
| 196 |
+
|
| 197 |
+
if histograms:
|
| 198 |
+
st.subheader("Histograms")
|
| 199 |
+
st.plotly_chart(combine_figures_as_subplots(histograms), use_container_width=True)
|
| 200 |
+
|
| 201 |
+
# 9. Line Plots
|
| 202 |
+
line_plots = []
|
| 203 |
+
if date_time_ls:
|
| 204 |
+
# Extract only date-related components
|
| 205 |
+
date_related_keywords = ['_year', '_month', '_day', '_weekday']
|
| 206 |
+
date_component_cols = [col for col in extracted_datetime if any(key in col for key in date_related_keywords)]
|
| 207 |
+
if date_component_cols:
|
| 208 |
+
st.subheader("Line Plots :-")
|
| 209 |
+
# Mapping of labels to values
|
| 210 |
+
time_grouping_options = {
|
| 211 |
+
"Daily": "D",
|
| 212 |
+
"Weekly": "W",
|
| 213 |
+
"Monthly": "ME",
|
| 214 |
+
"Yearly": "YE"
|
| 215 |
+
}
|
| 216 |
+
time_choice = st.pills("Choose time interval for grouping :- ", list(time_grouping_options.keys()))
|
| 217 |
+
# Extract the actual value for resampling
|
| 218 |
+
selected_freq = time_grouping_options[time_choice] if time_choice else "ME" # Use default "ME" if no selection
|
| 219 |
+
line_plots.extend(generate_line_plots(df, date_component_cols, continuous_ls, selected_freq))
|
| 220 |
+
|
| 221 |
+
if line_plots:
|
| 222 |
+
st.plotly_chart(combine_figures_as_subplots(line_plots), use_container_width= True)
|