'
+ return f''
def process_file_upload(self, file_obj, learning_type):
- """Enhanced file processing with safe datetime handling"""
if file_obj is None:
return "โ No file uploaded", "", [], gr.update(visible=False), ""
try:
file_path = file_obj.name
- file_name = os.path.basename(file_path)
- file_extension = os.path.splitext(file_name)[1].lower()
+ ext = os.path.splitext(file_path)[1].lower()
- if file_extension == '.csv':
+ if ext == '.csv':
df = pd.read_csv(file_path)
- file_type = 'csv'
- elif file_extension == '.json':
+ elif ext == '.json':
df = pd.read_json(file_path)
- file_type = 'json'
+ elif ext in ['.xlsx', '.xls']:
+ df = pd.read_excel(file_path)
else:
- return "โ Unsupported file type. Please upload CSV or JSON files only.", "", [], gr.update(visible=False), ""
-
- for col in df.columns:
- if df[col].dtype == 'object':
- try:
- pd.to_datetime(df[col], infer_datetime_format=True, errors='raise')
- df[col] = pd.to_datetime(df[col])
- except:
- pass
+ return "โ Unsupported file type. Use CSV, JSON, or Excel.", "", [], gr.update(visible=False), ""
self.current_data = df
- description, column_types = self.analyzer.safe_describe(df)
-
- file_size = os.path.getsize(file_path) / 1024
- memory_usage = df.memory_usage(deep=True).sum() / 1024**2
- missing_count = df.isnull().sum().sum()
- duplicate_count = df.duplicated().sum()
-
- preview_html = self._create_safe_data_preview(df)
+ missing = df.isnull().sum().sum()
+ dups = df.duplicated().sum()
+ numeric_cols = df.select_dtypes(include=[np.number]).columns
+ cat_cols = df.select_dtypes(include=['object']).columns
file_info = f"""
- Name: {file_name}
-Type: {file_type.upper()}
-Size: {file_size:.2f} KB
+Rows: {df.shape[0]:,}
-Columns: {df.shape[1]}
-Memory: {memory_usage:.2f} MB
+Missing: {missing_count:,} values
-Duplicates: {duplicate_count:,} rows
-Quality: {((1 - (missing_count + duplicate_count) / (df.shape[0] * df.shape[1])) * 100):.1f}%
-Numeric: {len(column_types['numeric'])}
-Categorical: {len(column_types['categorical'])}
-DateTime: {len(column_types['datetime'])}
+| {col} | " + table_html += "
|---|
| {cell} | " + table_html += "
| {col} | " - html += "
|---|
| {cell_value} | " - html += "
No target column โ skipped.
") - def _create_error_html(self, message): - return f""" -{message}
-End-to-end automated machine learning pipeline with comprehensive analysis
-Started: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
-+ Real multi-agent automated ML โ {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} +
Data loading information not available
" - - info = results.get('info', {}) + def _format_loading(self, result): + if not result: + return "No loading info.
" + info = result.get('info', {}) shape = info.get('shape', (0, 0)) - column_types = info.get('column_types', {}) - return f""" -Rows: {shape[0]:,}
-Columns: {shape[1]}
-Memory: {info.get('memory_usage', 'Unknown')}
+Numeric: {len(column_types.get('numeric', []))}
-Categorical: {len(column_types.get('categorical', []))}
-DateTime: {len(column_types.get('datetime', []))}
+โ Data loaded and column types detected successfully!
""" - def _format_data_cleaning_results(self, results): - """Format data cleaning results""" - if not results or results.get('status') != 'success': - return "Data cleaning information not available
" - - report = results.get('cleaning_report', {}) - duplicates = report.get('duplicates_removed', 0) - missing_values = report.get('missing_values', {}) + def _format_cleaning(self, result): + if not result: + return "No cleaning info.
" + report = result.get('cleaning_report', {}) + missing = report.get('missing_values', {}) + total_missing = sum(missing.values()) if isinstance(missing, dict) else 0 + dups = report.get('duplicates_removed', 0) outliers = report.get('outliers_handled', {}) - - total_missing = sum(missing_values.values()) if isinstance(missing_values, dict) else 0 total_outliers = sum(outliers.values()) if isinstance(outliers, dict) else 0 return f""" -Duplicates Removed: {duplicates}
-Missing Values: {total_missing}
-Outliers Handled: {total_outliers}
+โ Data cleaning completed successfully!
++ Final shape: {report.get('final_shape','N/A')} +
""" - def _create_dynamic_histogram(self, data, column): - """Create a dynamic histogram for a numeric column""" - try: - values = data[column].dropna() - if len(values) == 0: - return "No valid data for histogram
" - - # Dynamically adjust number of bins based on data size and spread - n_bins = min(max(int(np.sqrt(len(values))), 10), 50) - plt.figure(figsize=(8, 6)) - sns.histplot(values, bins=n_bins, kde=True, color='skyblue') - plt.title(f'Distribution of {column}', fontsize=14) - plt.xlabel(column, fontsize=12) - plt.ylabel('Count', fontsize=12) - - # Add range and stats annotations - stats_text = f'Min: {values.min():.2f}\nMax: {values.max():.2f}\nMean: {values.mean():.2f}' - plt.text(0.95, 0.95, stats_text, transform=plt.gca().transAxes, ha='right', va='top', - bbox=dict(facecolor='white', alpha=0.8)) - - html = self.create_plot_html(plt.gcf(), f"histogram_{column}") - plt.close() - - return f""" - {html} -Histogram showing the distribution of {column}
- """ - except Exception as e: - return f"Could not generate histogram for {column}: {str(e)}
" - - def _create_dynamic_bar(self, data, column, is_target=False): - """Create a dynamic bar plot for a categorical column""" - try: - value_counts = data[column].value_counts().head(10) # Limit to top 10 categories - labels = value_counts.index.tolist() - counts = value_counts.values.tolist() - - plt.figure(figsize=(8, 6)) - sns.barplot(x=counts, y=labels, palette='tab10') - plt.title(f"{'Target Distribution' if is_target else f'Distribution of {column}'}", fontsize=14) - plt.xlabel('Count', fontsize=12) - plt.ylabel(column, fontsize=12) - - # Add total count annotation - plt.text(0.95, 0.95, f'Total: {sum(counts)}', - transform=plt.gca().transAxes, ha='right', va='top', bbox=dict(facecolor='white', alpha=0.8)) - - html = self.create_plot_html(plt.gcf(), f"bar_{column}") - plt.close() - - return f""" - {html} -Bar plot showing the distribution of {column}
- """ - except Exception as e: - return f"Could not generate bar plot for {column}: {str(e)}
" - - def _create_dynamic_scatter(self, data, x_col, y_col, target=False): - """Create a dynamic scatter plot for regression analysis""" - try: - x_values = data[x_col].dropna() - y_values = data[y_col].dropna() - common_indices = x_values.index.intersection(y_values.index) - if len(common_indices) < 2: - return f"Not enough valid data for scatter plot between {x_col} and {y_col}
" - - x_values = x_values.loc[common_indices].head(1000) # Limit to 1000 points for performance - y_values = y_values.loc[common_indices].head(1000) - - plt.figure(figsize=(8, 6)) - plt.scatter(x_values, y_values, color='teal', alpha=0.6) - plt.title(f'{y_col} vs {x_col}', fontsize=14) - plt.xlabel(x_col, fontsize=12) - plt.ylabel(y_col, fontsize=12) - - # Add range and correlation annotations - corr = np.corrcoef(x_values, y_values)[0, 1] if len(x_values) > 1 else 0 - stats_text = f'X Range: {x_values.min():.2f} to {x_values.max():.2f}\nY Range: {y_values.min():.2f} to {y_values.max():.2f}\nCorr: {corr:.2f}' - plt.text(0.95, 0.95, stats_text, transform=plt.gca().transAxes, ha='right', va='top', - bbox=dict(facecolor='white', alpha=0.8)) - - html = self.create_plot_html(plt.gcf(), f"scatter_{x_col}_{y_col}") - plt.close() - - return f""" - {html} -Scatter plot showing relationship between {x_col} and {y_col}
- """ - except Exception as e: - return f"Could not generate scatter plot for {x_col} vs {y_col}: {str(e)}
" + def _format_eda(self, result, target_column): + if not result or result.get('status') != 'success': + return "EDA not available.
" - def _create_dynamic_correlation_heatmap(self, correlation_matrix): - """Create a dynamic correlation heatmap""" - try: - corr_df = pd.DataFrame(correlation_matrix) - if corr_df.empty or len(corr_df.columns) < 2: - return "Not enough numeric features for correlation analysis
" - - plt.figure(figsize=(min(10, len(corr_df.columns) * 1.2), min(8, len(corr_df.columns) * 1))) - sns.heatmap( - corr_df, - annot=True, - cmap='coolwarm', - vmin=-1, - vmax=1, - center=0, - square=True, - fmt='.2f', - annot_kws={'size': max(8, 12 - len(corr_df.columns) // 2)}, - cbar_kws={'label': 'Correlation Coefficient'} - ) - plt.title('Correlation Matrix Heatmap', fontsize=14, pad=15) - plt.xticks(rotation=45, ha='right') - plt.yticks(rotation=0) - - html = self.create_plot_html(plt.gcf(), "correlation_heatmap") - plt.close() - - return f""" - {html} -Heatmap showing correlations between numeric features
- """ - except Exception as e: - return f"Could not generate correlation heatmap: {str(e)}
" - - def _format_eda_results(self, results, data, learning_type=None, target_column=None): - """Format EDA results with dynamic visualizations""" - if not results or results.get('status') != 'success' or data is None: - return "EDA information not available or no data loaded
" - - analysis = results.get('analysis', {}) - column_types = analysis.get('column_types', {}) - correlations = analysis.get('correlations', {}) + analysis = result.get('analysis', {}) + col_types = analysis.get('column_types', {}) + quality = analysis.get('data_quality_score', 'N/A') html = f""" -Numeric Features: {len(column_types.get('numeric', []))}
-Categorical Features: {len(column_types.get('categorical', []))}
-DateTime Features: {len(column_types.get('datetime', []))}
+โ Exploratory Data Analysis completed!
- """ + # Correlation heatmap + corr_data = analysis.get('correlations', {}).get('matrix', {}) + if corr_data and len(corr_data) > 1: + try: + corr_df = pd.DataFrame(corr_data) + fig, ax = plt.subplots(figsize=(min(10, len(corr_df)*1.2), + min(8, len(corr_df)))) + sns.heatmap(corr_df, annot=True, cmap='coolwarm', center=0, + vmin=-1, vmax=1, fmt='.2f', ax=ax, + annot_kws={'size': max(7, 11 - len(corr_df)//2)}) + ax.set_title('Correlation Matrix', fontsize=13) + plt.xticks(rotation=45, ha='right') + plt.tight_layout() + html += self.create_plot_html(fig, 'corr_heatmap') + except: + pass + + # Numeric distributions + if self.current_data is not None: + numeric_cols = col_types.get('numeric', [])[:3] + for col in numeric_cols: + try: + vals = self.current_data[col].dropna() + fig, ax = plt.subplots(figsize=(8, 4)) + sns.histplot(vals, bins=min(30, int(np.sqrt(len(vals)))), + kde=True, color='steelblue', ax=ax) + ax.set_title(f'Distribution of {col}', fontsize=12) + ax.set_xlabel(col) + ax.set_ylabel('Count') + plt.tight_layout() + html += self.create_plot_html(fig, f'hist_{col}') + except: + pass + + # Categorical bar plots + cat_cols = col_types.get('categorical', [])[:2] + for col in cat_cols: + try: + vc = self.current_data[col].value_counts().head(10) + fig, ax = plt.subplots(figsize=(8, 4)) + sns.barplot(x=vc.values, y=vc.index.astype(str), + palette='viridis', ax=ax) + ax.set_title(f'Top values in {col}', fontsize=12) + plt.tight_layout() + html += self.create_plot_html(fig, f'bar_{col}') + except: + pass + + # Target analysis + ta = analysis.get('target_analysis', {}) + if ta and target_column and target_column in self.current_data.columns: + try: + if ta['type'] == 'classification': + dist = self.current_data[target_column].value_counts() + fig, ax = plt.subplots(figsize=(7, 4)) + sns.barplot(x=dist.values, y=dist.index.astype(str), + palette='Set2', ax=ax) + ax.set_title(f'Target Distribution: {target_column}', fontsize=12) + plt.tight_layout() + html += self.create_plot_html(fig, 'target_dist') + else: + fig, ax = plt.subplots(figsize=(8, 4)) + sns.histplot(self.current_data[target_column].dropna(), + kde=True, color='coral', ax=ax) + ax.set_title(f'Target Distribution: {target_column}', fontsize=12) + plt.tight_layout() + html += self.create_plot_html(fig, 'target_dist') + except: + pass + + # Strong correlations + strong = analysis.get('correlations', {}).get('strong', []) + if strong: + html += f"๐ Strong Correlations Found ({len(strong)}):
Domain analysis information not available
" - - domain = results.get('detected_domain', 'general') - insights = results.get('insights', []) - recommendations = results.get('recommendations', []) + def _format_domain(self, result): + if not result: + return "No domain info.
" + domain = result.get('detected_domain', 'general') + insights = result.get('insights', []) + recommendations = result.get('recommendations', []) return f""" -โ Domain analysis completed!
""" - def _format_modeling_results(self, results, enable_deep_learning): - """Format modeling results with visualizations""" - if not results or results.get('status') != 'success': - return "Modeling information not available
" + def _format_modeling(self, result): + if not result or result.get('status') != 'success': + return f"Modeling failed: {result.get('error','Unknown')}
" - problem_type = results.get('problem_type', 'unknown') - best_model = results.get('best_model', 'Unknown') - model_results = results.get('results', {}) - feature_importance = results.get('feature_importance', {}) + problem_type = result.get('problem_type', 'unknown') + best_name = result.get('best_model', 'N/A') + results = result.get('results', {}) + feature_imp = result.get('feature_importance', {}) + # Results table html = f""" -| Model | -- {'Accuracy' if problem_type == 'classification' else 'RMSE'} - | -- {'F1 Score' if problem_type == 'classification' else 'Rยฒ Score'} - | -
|---|---|---|
| {model} | -{metric1:.3f} | -{metric2:.3f} | -
Deep learning models were evaluated but not included in final results due to complexity constraints.
-โ Model training and evaluation completed!
+Problem Type: {problem_type.title()}
| Model | """ + + if 'classification' in problem_type: + html += """Accuracy | +F1 Score | +Precision | +Recall | """ + else: + html += """RMSE | +MAE | +Rยฒ Score | """ - plt.figure(figsize=(8, max(6, len(features) * 0.5))) - sns.barplot(x=importances, y=features, palette='viridis') - plt.title('Feature Importance', fontsize=14) - plt.xlabel('Importance Score', fontsize=12) - plt.ylabel('Features', fontsize=12) + html += "
|---|---|---|---|---|---|---|---|
| {name} {'๐' if name == best_name else ''} | " + + if 'classification' in problem_type: + html += f"{metrics.get('accuracy','N/A')} | " + html += f"{metrics.get('f1_score','N/A')} | " + html += f"{metrics.get('precision','N/A')} | " + html += f"{metrics.get('recall','N/A')} | " + else: + html += f"{metrics.get('rmse','N/A')} | " + html += f"{metrics.get('mae','N/A')} | " + html += f"{metrics.get('r2_score','N/A')} | " - html = self.create_plot_html(plt.gcf(), "feature_importance") - plt.close() + html += "
Bar plot showing feature importance scores
- """ - except Exception as e: - return f"Could not generate feature importance plot: {str(e)}
" + # Feature importance plot + if feature_imp: + try: + features = list(feature_imp.keys())[:15] + importances = [feature_imp[f] for f in features] + fig, ax = plt.subplots(figsize=(8, max(5, len(features)*0.4))) + colors = ['#e74c3c' if f == features[0] else '#3498db' for f in features] + bars = ax.barh(features[::-1], importances[::-1], color=colors[::-1]) + ax.set_title(f'Feature Importance โ {best_name}', fontsize=13) + ax.set_xlabel('Importance Score') + for bar, imp in zip(bars, importances[::-1]): + ax.text(imp + 0.001, bar.get_y() + bar.get_height()/2, + f'{imp:.3f}', va='center', fontsize=9) + plt.tight_layout() + html += self.create_plot_html(fig, 'feature_importance') + except: + pass + + # Model comparison bar chart + if valid: + try: + names = list(valid.keys()) + if 'classification' in problem_type: + scores = [valid[n].get('accuracy', 0) for n in names] + metric_label = 'Accuracy' + else: + scores = [valid[n].get('r2_score', 0) for n in names] + metric_label = 'Rยฒ Score' + + fig, ax = plt.subplots(figsize=(10, 5)) + colors = ['#e74c3c' if n == best_name else '#3498db' for n in names] + bars = ax.bar(range(len(names)), scores, color=colors) + ax.set_xticks(range(len(names))) + ax.set_xticklabels(names, rotation=45, ha='right', fontsize=9) + ax.set_ylabel(metric_label) + ax.set_title(f'Model Comparison โ {metric_label}', fontsize=13) + for bar, score in zip(bars, scores): + ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.005, + f'{score:.3f}', ha='center', fontsize=8) + plt.tight_layout() + html += self.create_plot_html(fig, 'model_comparison') + except: + pass - def _format_unsupervised_results(self, data): - """Format unsupervised analysis results with dynamic clustering visualization""" - if data is None: - return "No data available for unsupervised analysis
" + return html - column_types = self.analyzer.detect_column_types(data) - numeric_cols = column_types['numeric'] + def _format_clustering(self): + if self.current_data is None: + return "No data available.
" - html = """ -Performed clustering analysis to identify natural groupings in the data.
- """ + html = "Unsupervised analysis โ performing KMeans clustering.
" + numeric_cols = self.current_data.select_dtypes(include=[np.number]).columns.tolist() if len(numeric_cols) >= 2: try: - # Perform KMeans clustering with dynamic number of clusters - X = data[numeric_cols].dropna().head(1000) - n_clusters = min(3, len(X) // 10) if len(X) > 10 else 2 - kmeans = KMeans(n_clusters=n_clusters, random_state=42) - clusters = kmeans.fit_predict(X) - - plt.figure(figsize=(8, 6)) - plt.scatter(X.iloc[:, 0], X.iloc[:, 1], c=clusters, cmap='viridis', alpha=0.6) - plt.title(f'Clustering: {numeric_cols[0]} vs {numeric_cols[1]}', fontsize=14) - plt.xlabel(numeric_cols[0], fontsize=12) - plt.ylabel(numeric_cols[1], fontsize=12) - - # Add cluster count annotation - plt.text(0.95, 0.95, f'Clusters: {n_clusters}', - transform=plt.gca().transAxes, ha='right', va='top', - bbox=dict(facecolor='white', alpha=0.8)) - - html += self.create_plot_html(plt.gcf(), "clustering_plot") - plt.close() - - html += f""" -- Scatter plot showing clusters based on {numeric_cols[0]} and {numeric_cols[1]} -
- """ + X = self.current_data[numeric_cols].dropna().head(1000) + scaler = StandardScaler() + X_scaled = scaler.fit_transform(X) + n_clusters = min(4, max(2, len(X) // 50)) + kmeans = KMeans(n_clusters=n_clusters, random_state=42, n_init=10) + labels = kmeans.fit_predict(X_scaled) + sil = round(silhouette_score(X_scaled, labels), 3) + + fig, ax = plt.subplots(figsize=(8, 6)) + scatter = ax.scatter(X.iloc[:, 0], X.iloc[:, 1], + c=labels, cmap='viridis', alpha=0.6) + plt.colorbar(scatter, ax=ax) + ax.set_title(f'KMeans Clustering (k={n_clusters}, silhouette={sil})', fontsize=12) + ax.set_xlabel(numeric_cols[0]) + ax.set_ylabel(numeric_cols[1]) + plt.tight_layout() + html += self.create_plot_html(fig, 'clustering') + html += f"Silhouette Score: {sil} (higher is better, max=1.0)
" except Exception as e: - html += f"Could not generate clustering plot: {str(e)}
" - else: - html += "Not enough numeric columns for clustering visualization
" + html += f"Could not generate clustering plot: {e}
" - html += """ -โ Unsupervised analysis completed!
-- Analysis Type: {learning_type} | Domain: {domain or 'General'} | - Deep Learning: {'Enabled' if enable_deep_learning else 'Disabled'} | - AutoML: {'Enabled' if enable_automl else 'Disabled'} +
+ Domain: {domain or 'General'} | + Completed: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} +
++ All model results are REAL โ trained on your actual data!
-Completed: {completion_time}
{msg}
โ Final results compiled!
- """ - return html - - def generate_report(self): - """Generate a downloadable HTML report with all results and visualizations""" - if not self.pipeline_results: - return self._create_error_html("No pipeline results available to generate report.") - - html = f""" - - - -