Spaces:
Sleeping
Sleeping
Initial deploy: ML Data Analysis Studio
Browse files- README.md +38 -7
- app.py +305 -0
- chat_engine.py +164 -0
- data_processor.py +212 -0
- ml_models.py +271 -0
- report_generator.py +153 -0
- requirements.txt +8 -0
README.md
CHANGED
|
@@ -1,13 +1,44 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.13'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: ML Data Analysis Studio
|
| 3 |
+
emoji: 🤖
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: indigo
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 4.44.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
+
license: mit
|
| 11 |
---
|
| 12 |
|
| 13 |
+
# 🤖 ML Data Analysis Studio
|
| 14 |
+
|
| 15 |
+
An interactive machine-learning playground. Upload a dataset in almost any format, clean it, train models, inspect the generated code, and download full analysis reports.
|
| 16 |
+
|
| 17 |
+
## Features
|
| 18 |
+
|
| 19 |
+
- **💬 ML Assistant** — chat about ML models (random forests, SVM, overfitting, metrics, model selection...) and get guided through the analysis workflow.
|
| 20 |
+
- **📊 Data upload & preprocessing** — supports CSV, TSV, Excel, JSON, and Parquet. Automatic profiling (missing values, duplicates, dtypes), then configurable cleaning: imputation or row-dropping, duplicate removal, one-hot encoding, and standard scaling. Cleaned data is cached temporarily and reused for modeling.
|
| 21 |
+
- **🧠 Model training** — regression (Linear, Decision Tree, Random Forest, Gradient Boosting, SVR, KNN) and classification (Logistic, Decision Tree, Random Forest, Gradient Boosting, SVC, KNN). Auto-suggests the task type from your target column. Shows metrics, an actual-vs-predicted / confusion-matrix plot, and feature importance.
|
| 22 |
+
- **👨💻 Code viewer** — dropdown accordions reveal the exact standalone Python (pandas + scikit-learn) equivalent to every preprocessing and training step, ready to copy.
|
| 23 |
+
- **📥 Downloads** — one-click export of a PDF report, a self-contained HTML report, and the cleaned dataset as CSV.
|
| 24 |
+
|
| 25 |
+
## Deploying to Hugging Face Spaces
|
| 26 |
+
|
| 27 |
+
1. Create a new Space at https://huggingface.co/new-space → SDK: **Gradio**.
|
| 28 |
+
2. Upload all files in this folder (`app.py`, `chat_engine.py`, `data_processor.py`, `ml_models.py`, `report_generator.py`, `requirements.txt`, `README.md`).
|
| 29 |
+
3. The Space builds automatically and goes live in a few minutes.
|
| 30 |
+
|
| 31 |
+
Or via git:
|
| 32 |
+
|
| 33 |
+
```bash
|
| 34 |
+
git clone https://huggingface.co/spaces/<your-username>/<space-name>
|
| 35 |
+
cp -r ml-analysis-space/* <space-name>/
|
| 36 |
+
cd <space-name> && git add . && git commit -m "Initial app" && git push
|
| 37 |
+
```
|
| 38 |
+
|
| 39 |
+
## Run locally
|
| 40 |
+
|
| 41 |
+
```bash
|
| 42 |
+
pip install -r requirements.txt
|
| 43 |
+
python app.py
|
| 44 |
+
```
|
app.py
ADDED
|
@@ -0,0 +1,305 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ML Data Analysis Studio — Gradio app for Hugging Face Spaces.
|
| 2 |
+
|
| 3 |
+
Chat about ML models, upload data in any common format, preprocess it,
|
| 4 |
+
train regression/classification models, inspect the generated code,
|
| 5 |
+
and download PDF/HTML reports plus the cleaned dataset.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
import os
|
| 9 |
+
import traceback
|
| 10 |
+
|
| 11 |
+
import gradio as gr
|
| 12 |
+
import pandas as pd
|
| 13 |
+
|
| 14 |
+
import chat_engine
|
| 15 |
+
import ml_models
|
| 16 |
+
from data_processor import (
|
| 17 |
+
CLEANED_PATH,
|
| 18 |
+
generate_preprocessing_code,
|
| 19 |
+
load_data,
|
| 20 |
+
preprocess_data,
|
| 21 |
+
profile_data,
|
| 22 |
+
profile_text,
|
| 23 |
+
)
|
| 24 |
+
from report_generator import generate_html_report, generate_pdf_report
|
| 25 |
+
|
| 26 |
+
# ---------------------------------------------------------------- state ----
|
| 27 |
+
STATE = {
|
| 28 |
+
"raw_df": None,
|
| 29 |
+
"clean_df": None,
|
| 30 |
+
"file_name": None,
|
| 31 |
+
"profile": None,
|
| 32 |
+
"steps": [],
|
| 33 |
+
"result": None,
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ----------------------------------------------------------------- chat ----
|
| 38 |
+
def chat_fn(message, history):
|
| 39 |
+
if not message.strip():
|
| 40 |
+
return ""
|
| 41 |
+
return chat_engine.respond(message, STATE["file_name"])
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
# --------------------------------------------------------------- upload ----
|
| 45 |
+
def upload_fn(file):
|
| 46 |
+
if file is None:
|
| 47 |
+
return "Upload a file to begin.", None, gr.update(choices=[]), ""
|
| 48 |
+
try:
|
| 49 |
+
df = load_data(file.name)
|
| 50 |
+
except Exception as e:
|
| 51 |
+
return f"❌ Could not read file: {e}", None, gr.update(choices=[]), ""
|
| 52 |
+
STATE["raw_df"] = df
|
| 53 |
+
STATE["file_name"] = os.path.basename(file.name)
|
| 54 |
+
STATE["profile"] = profile_data(df)
|
| 55 |
+
STATE["clean_df"] = None
|
| 56 |
+
STATE["result"] = None
|
| 57 |
+
msg = (
|
| 58 |
+
f"✅ Loaded **{STATE['file_name']}**. Do you want me to analyse this data? "
|
| 59 |
+
f"Review the profile below, adjust the cleaning options, and click "
|
| 60 |
+
f"**Run Preprocessing**.\n\n" + profile_text(STATE["profile"])
|
| 61 |
+
)
|
| 62 |
+
cols = df.columns.tolist()
|
| 63 |
+
return msg, df.head(20), gr.update(choices=cols, value=cols[-1]), ""
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
# ----------------------------------------------------------- preprocess ----
|
| 67 |
+
def preprocess_fn(missing_strategy, drop_dups, encode_cats, scale_num, target):
|
| 68 |
+
if STATE["raw_df"] is None:
|
| 69 |
+
return "⚠️ Upload a dataset first.", None, "", gr.update(choices=[])
|
| 70 |
+
try:
|
| 71 |
+
clean_df, steps = preprocess_data(
|
| 72 |
+
STATE["raw_df"],
|
| 73 |
+
missing_strategy=missing_strategy,
|
| 74 |
+
drop_duplicates=drop_dups,
|
| 75 |
+
encode_categoricals=encode_cats,
|
| 76 |
+
scale_numeric=scale_num,
|
| 77 |
+
target_column=target or None,
|
| 78 |
+
)
|
| 79 |
+
except Exception as e:
|
| 80 |
+
return f"❌ Preprocessing failed: {e}", None, "", gr.update(choices=[])
|
| 81 |
+
STATE["clean_df"] = clean_df
|
| 82 |
+
STATE["steps"] = steps
|
| 83 |
+
code = generate_preprocessing_code(
|
| 84 |
+
missing_strategy, drop_dups, encode_cats, scale_num, target or None
|
| 85 |
+
)
|
| 86 |
+
msg = "### ✅ Preprocessing complete\n\n" + "\n".join(f"- {s}" for s in steps)
|
| 87 |
+
cols = clean_df.columns.tolist()
|
| 88 |
+
default_target = target if target in cols else (cols[-1] if cols else None)
|
| 89 |
+
return msg, clean_df.head(20), code, gr.update(choices=cols, value=default_target)
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# ---------------------------------------------------------------- train ----
|
| 93 |
+
def suggest_task_fn(target):
|
| 94 |
+
df = STATE["clean_df"] if STATE["clean_df"] is not None else STATE["raw_df"]
|
| 95 |
+
if df is None or not target or target not in df.columns:
|
| 96 |
+
return gr.update()
|
| 97 |
+
task = ml_models.suggest_task(df, target)
|
| 98 |
+
return gr.update(
|
| 99 |
+
value=task,
|
| 100 |
+
info=f"Suggested: {task} (based on the '{target}' column)",
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def task_models_fn(task):
|
| 105 |
+
models = (
|
| 106 |
+
list(ml_models.CLASSIFICATION_MODELS)
|
| 107 |
+
if task == "Classification"
|
| 108 |
+
else list(ml_models.REGRESSION_MODELS)
|
| 109 |
+
)
|
| 110 |
+
return gr.update(choices=models, value=models[0])
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def train_fn(target, task, model_name, test_size):
|
| 114 |
+
empty = (None,) * 4
|
| 115 |
+
if STATE["clean_df"] is None:
|
| 116 |
+
if STATE["raw_df"] is None:
|
| 117 |
+
return ("⚠️ Upload a dataset first (Data & Preprocessing tab).", *empty)
|
| 118 |
+
return ("⚠️ Run preprocessing first (Data & Preprocessing tab).", *empty)
|
| 119 |
+
if not target:
|
| 120 |
+
return ("⚠️ Select a target column.", *empty)
|
| 121 |
+
valid = (
|
| 122 |
+
ml_models.CLASSIFICATION_MODELS
|
| 123 |
+
if task == "Classification"
|
| 124 |
+
else ml_models.REGRESSION_MODELS
|
| 125 |
+
)
|
| 126 |
+
if model_name not in valid:
|
| 127 |
+
model_name = list(valid)[0]
|
| 128 |
+
try:
|
| 129 |
+
result = ml_models.train_model(
|
| 130 |
+
STATE["clean_df"], target, model_name, task, test_size
|
| 131 |
+
)
|
| 132 |
+
except Exception as e:
|
| 133 |
+
traceback.print_exc()
|
| 134 |
+
return (f"❌ Training failed: {e}", *empty)
|
| 135 |
+
STATE["result"] = result
|
| 136 |
+
|
| 137 |
+
metrics_md = "\n".join(f"| {k} | {v} |" for k, v in result["metrics"].items())
|
| 138 |
+
msg = (
|
| 139 |
+
f"### ✅ {model_name} trained\n\n"
|
| 140 |
+
f"**Task:** {task} | **Target:** `{target}` | "
|
| 141 |
+
f"**Train/test:** {result['n_train']:,}/{result['n_test']:,} | "
|
| 142 |
+
f"**Features used:** {len(result['features'])}\n\n"
|
| 143 |
+
f"| Metric | Value |\n|---|---|\n{metrics_md}"
|
| 144 |
+
)
|
| 145 |
+
code = ml_models.generate_model_code(model_name, task, target, test_size)
|
| 146 |
+
return (
|
| 147 |
+
msg,
|
| 148 |
+
result["plot_path"],
|
| 149 |
+
result["importance_path"],
|
| 150 |
+
code,
|
| 151 |
+
pd.DataFrame([result["metrics"]]),
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# ------------------------------------------------------------- download ----
|
| 156 |
+
def download_fn(formats):
|
| 157 |
+
if STATE["result"] is None:
|
| 158 |
+
return "⚠️ Train a model first — the report includes its results.", []
|
| 159 |
+
files = []
|
| 160 |
+
try:
|
| 161 |
+
if "PDF report" in formats:
|
| 162 |
+
files.append(generate_pdf_report(STATE["profile"], STATE["steps"], STATE["result"]))
|
| 163 |
+
if "HTML report" in formats:
|
| 164 |
+
files.append(generate_html_report(STATE["profile"], STATE["steps"], STATE["result"]))
|
| 165 |
+
if "Cleaned data (CSV)" in formats and os.path.exists(CLEANED_PATH):
|
| 166 |
+
files.append(CLEANED_PATH)
|
| 167 |
+
except Exception as e:
|
| 168 |
+
traceback.print_exc()
|
| 169 |
+
return f"❌ Report generation failed: {e}", []
|
| 170 |
+
if not files:
|
| 171 |
+
return "⚠️ Select at least one format.", []
|
| 172 |
+
return f"✅ Generated {len(files)} file(s) — download below.", files
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
# ------------------------------------------------------------------ UI ----
|
| 176 |
+
CSS = """
|
| 177 |
+
.gradio-container { max-width: 1100px !important; }
|
| 178 |
+
footer { display: none !important; }
|
| 179 |
+
"""
|
| 180 |
+
|
| 181 |
+
with gr.Blocks(title="ML Data Analysis Studio", css=CSS, theme=gr.themes.Soft()) as demo:
|
| 182 |
+
gr.Markdown(
|
| 183 |
+
"# 🤖 ML Data Analysis Studio\n"
|
| 184 |
+
"Chat about machine learning, upload your data, clean it, train models, "
|
| 185 |
+
"inspect the code, and download reports."
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
with gr.Tabs():
|
| 189 |
+
# ---------------------------------------------------- Tab 1: Chat
|
| 190 |
+
with gr.Tab("💬 ML Assistant"):
|
| 191 |
+
gr.ChatInterface(
|
| 192 |
+
fn=chat_fn,
|
| 193 |
+
chatbot=gr.Chatbot(
|
| 194 |
+
value=[{"role": "assistant", "content": chat_engine.WELCOME}],
|
| 195 |
+
height=420,
|
| 196 |
+
type="messages",
|
| 197 |
+
),
|
| 198 |
+
type="messages",
|
| 199 |
+
examples=[
|
| 200 |
+
"What do you want to analyse today?",
|
| 201 |
+
"Which model should I use?",
|
| 202 |
+
"What is a random forest?",
|
| 203 |
+
"Explain overfitting",
|
| 204 |
+
"What do precision and recall mean?",
|
| 205 |
+
],
|
| 206 |
+
)
|
| 207 |
+
|
| 208 |
+
# -------------------------------------- Tab 2: Data & Preprocessing
|
| 209 |
+
with gr.Tab("📊 Data & Preprocessing"):
|
| 210 |
+
gr.Markdown("### Step 1 — Upload your dataset (CSV, TSV, Excel, JSON, or Parquet)")
|
| 211 |
+
file_input = gr.File(
|
| 212 |
+
label="Upload data",
|
| 213 |
+
file_types=[".csv", ".tsv", ".xlsx", ".xls", ".json", ".parquet", ".txt"],
|
| 214 |
+
)
|
| 215 |
+
upload_status = gr.Markdown("Upload a file to begin.")
|
| 216 |
+
data_preview = gr.Dataframe(label="Data preview (first 20 rows)")
|
| 217 |
+
|
| 218 |
+
gr.Markdown("### Step 2 — Preprocessing & cleaning options")
|
| 219 |
+
with gr.Row():
|
| 220 |
+
missing_dd = gr.Dropdown(
|
| 221 |
+
["Impute (mean/mode)", "Drop rows with missing values"],
|
| 222 |
+
value="Impute (mean/mode)",
|
| 223 |
+
label="Missing values",
|
| 224 |
+
)
|
| 225 |
+
target_dd_pre = gr.Dropdown(
|
| 226 |
+
[], label="Target column (kept out of encoding/scaling)"
|
| 227 |
+
)
|
| 228 |
+
with gr.Row():
|
| 229 |
+
dups_cb = gr.Checkbox(True, label="Remove duplicate rows")
|
| 230 |
+
encode_cb = gr.Checkbox(True, label="One-hot encode categoricals")
|
| 231 |
+
scale_cb = gr.Checkbox(False, label="Scale numeric features (StandardScaler)")
|
| 232 |
+
preprocess_btn = gr.Button("🧹 Run Preprocessing", variant="primary")
|
| 233 |
+
preprocess_status = gr.Markdown()
|
| 234 |
+
clean_preview = gr.Dataframe(label="Cleaned data preview")
|
| 235 |
+
with gr.Accordion("👨💻 View preprocessing code", open=False):
|
| 236 |
+
preprocess_code = gr.Code(language="python")
|
| 237 |
+
|
| 238 |
+
# ------------------------------------------- Tab 3: Model Training
|
| 239 |
+
with gr.Tab("🧠 Model Training"):
|
| 240 |
+
gr.Markdown("### Step 3 — Configure and train a model")
|
| 241 |
+
with gr.Row():
|
| 242 |
+
target_dd = gr.Dropdown([], label="Target column (what to predict)")
|
| 243 |
+
task_radio = gr.Radio(
|
| 244 |
+
["Regression", "Classification"], value="Regression", label="Task"
|
| 245 |
+
)
|
| 246 |
+
with gr.Row():
|
| 247 |
+
model_dd = gr.Dropdown(
|
| 248 |
+
list(ml_models.REGRESSION_MODELS),
|
| 249 |
+
value="Linear Regression",
|
| 250 |
+
label="Model",
|
| 251 |
+
)
|
| 252 |
+
test_slider = gr.Slider(0.1, 0.4, 0.2, step=0.05, label="Test set fraction")
|
| 253 |
+
train_btn = gr.Button("🚀 Train Model", variant="primary")
|
| 254 |
+
train_status = gr.Markdown()
|
| 255 |
+
with gr.Row():
|
| 256 |
+
result_plot = gr.Image(label="Result plot", type="filepath")
|
| 257 |
+
importance_plot = gr.Image(label="Feature importance", type="filepath")
|
| 258 |
+
metrics_df = gr.Dataframe(label="Metrics", visible=True)
|
| 259 |
+
with gr.Accordion("👨💻 View model training code", open=False):
|
| 260 |
+
model_code = gr.Code(language="python")
|
| 261 |
+
|
| 262 |
+
# ----------------------------------------------- Tab 4: Downloads
|
| 263 |
+
with gr.Tab("📥 Reports & Downloads"):
|
| 264 |
+
gr.Markdown(
|
| 265 |
+
"### Step 4 — Export your analysis\n"
|
| 266 |
+
"Generates a report with the dataset profile, preprocessing steps, "
|
| 267 |
+
"model results, and charts."
|
| 268 |
+
)
|
| 269 |
+
formats_cg = gr.CheckboxGroup(
|
| 270 |
+
["PDF report", "HTML report", "Cleaned data (CSV)"],
|
| 271 |
+
value=["PDF report", "HTML report", "Cleaned data (CSV)"],
|
| 272 |
+
label="Formats",
|
| 273 |
+
)
|
| 274 |
+
report_btn = gr.Button("📄 Generate Downloads", variant="primary")
|
| 275 |
+
report_status = gr.Markdown()
|
| 276 |
+
report_files = gr.Files(label="Your downloads")
|
| 277 |
+
|
| 278 |
+
# ------------------------------------------------------------ wiring ----
|
| 279 |
+
file_input.change(
|
| 280 |
+
upload_fn,
|
| 281 |
+
inputs=file_input,
|
| 282 |
+
outputs=[upload_status, data_preview, target_dd_pre, preprocess_status],
|
| 283 |
+
).then(lambda t: gr.update(choices=STATE["raw_df"].columns.tolist() if STATE["raw_df"] is not None else [], value=t),
|
| 284 |
+
inputs=target_dd_pre, outputs=target_dd)
|
| 285 |
+
|
| 286 |
+
preprocess_btn.click(
|
| 287 |
+
preprocess_fn,
|
| 288 |
+
inputs=[missing_dd, dups_cb, encode_cb, scale_cb, target_dd_pre],
|
| 289 |
+
outputs=[preprocess_status, clean_preview, preprocess_code, target_dd],
|
| 290 |
+
)
|
| 291 |
+
|
| 292 |
+
target_dd.change(suggest_task_fn, inputs=target_dd, outputs=task_radio)
|
| 293 |
+
task_radio.change(task_models_fn, inputs=task_radio, outputs=model_dd)
|
| 294 |
+
|
| 295 |
+
train_btn.click(
|
| 296 |
+
train_fn,
|
| 297 |
+
inputs=[target_dd, task_radio, model_dd, test_slider],
|
| 298 |
+
outputs=[train_status, result_plot, importance_plot, model_code, metrics_df],
|
| 299 |
+
)
|
| 300 |
+
|
| 301 |
+
report_btn.click(download_fn, inputs=formats_cg, outputs=[report_status, report_files])
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
if __name__ == "__main__":
|
| 305 |
+
demo.launch()
|
chat_engine.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Rule-based chat assistant: ML knowledge base + guided analysis flow."""
|
| 2 |
+
|
| 3 |
+
WELCOME = (
|
| 4 |
+
"Hi! I'm your ML analysis assistant. Ask me anything about machine learning "
|
| 5 |
+
"models (e.g. *'What is a random forest?'*, *'When should I use logistic "
|
| 6 |
+
"regression?'*), or upload a dataset in the **Data & Preprocessing** tab.\n\n"
|
| 7 |
+
"**So — what do you want to analyse today?**"
|
| 8 |
+
)
|
| 9 |
+
|
| 10 |
+
KNOWLEDGE_BASE = {
|
| 11 |
+
("linear regression",): (
|
| 12 |
+
"**Linear Regression** fits a straight line (or hyperplane) that minimizes the "
|
| 13 |
+
"squared error between predictions and actual values. Use it when the target is "
|
| 14 |
+
"continuous and roughly linearly related to the features. Pros: fast, interpretable "
|
| 15 |
+
"coefficients. Cons: can't capture non-linear patterns, sensitive to outliers."
|
| 16 |
+
),
|
| 17 |
+
("logistic regression",): (
|
| 18 |
+
"**Logistic Regression** is a classification model that predicts class "
|
| 19 |
+
"probabilities via the sigmoid function. Great baseline for binary classification. "
|
| 20 |
+
"Pros: fast, well-calibrated probabilities, interpretable. Cons: linear decision "
|
| 21 |
+
"boundary only."
|
| 22 |
+
),
|
| 23 |
+
("random forest",): (
|
| 24 |
+
"**Random Forest** builds many decision trees on bootstrapped samples with random "
|
| 25 |
+
"feature subsets, then averages (regression) or votes (classification). Pros: "
|
| 26 |
+
"handles non-linearity, robust to outliers/overfitting, gives feature importance. "
|
| 27 |
+
"Cons: slower and less interpretable than a single tree."
|
| 28 |
+
),
|
| 29 |
+
("decision tree",): (
|
| 30 |
+
"**Decision Trees** split data by feature thresholds to form a flowchart of rules. "
|
| 31 |
+
"Pros: highly interpretable, no scaling needed, handles mixed data. Cons: prone to "
|
| 32 |
+
"overfitting — usually better inside an ensemble (Random Forest / Gradient Boosting)."
|
| 33 |
+
),
|
| 34 |
+
("gradient boosting", "xgboost", "boosting"): (
|
| 35 |
+
"**Gradient Boosting** builds trees sequentially, each one correcting the errors of "
|
| 36 |
+
"the previous ones. Often the top performer on tabular data (XGBoost/LightGBM are "
|
| 37 |
+
"popular variants). Pros: high accuracy. Cons: more hyperparameters, slower to train."
|
| 38 |
+
),
|
| 39 |
+
("svm", "support vector"): (
|
| 40 |
+
"**Support Vector Machines** find the boundary that maximizes the margin between "
|
| 41 |
+
"classes (SVC) or fit within an error tube (SVR). Kernels enable non-linear "
|
| 42 |
+
"boundaries. Pros: effective in high dimensions. Cons: slow on large datasets, "
|
| 43 |
+
"needs feature scaling."
|
| 44 |
+
),
|
| 45 |
+
("knn", "k-nearest", "nearest neighbor"): (
|
| 46 |
+
"**K-Nearest Neighbors** predicts from the K closest training points — majority "
|
| 47 |
+
"vote for classification, average for regression. Pros: simple, no training phase. "
|
| 48 |
+
"Cons: slow at prediction time, sensitive to scaling and irrelevant features."
|
| 49 |
+
),
|
| 50 |
+
("overfitting", "overfit"): (
|
| 51 |
+
"**Overfitting** is when a model memorizes training data (including noise) and "
|
| 52 |
+
"performs poorly on new data. Signs: high train accuracy, low test accuracy. "
|
| 53 |
+
"Remedies: more data, simpler models, regularization, cross-validation, early stopping."
|
| 54 |
+
),
|
| 55 |
+
("classification", "classify"): (
|
| 56 |
+
"**Classification** predicts a discrete category (spam/not-spam, species, churn). "
|
| 57 |
+
"Models here: Logistic Regression, Decision Tree, Random Forest, Gradient Boosting, "
|
| 58 |
+
"SVC, and KNN. Metrics: accuracy, precision, recall, F1."
|
| 59 |
+
),
|
| 60 |
+
("regression", "predict a number", "continuous"): (
|
| 61 |
+
"**Regression** predicts a continuous number (price, temperature, sales). Models "
|
| 62 |
+
"here: Linear Regression, Decision Tree, Random Forest, Gradient Boosting, SVR, and "
|
| 63 |
+
"KNN. Metrics: R², MAE, RMSE."
|
| 64 |
+
),
|
| 65 |
+
("preprocessing", "cleaning", "clean data", "preprocess"): (
|
| 66 |
+
"**Preprocessing** in this app: drop empty columns, remove duplicates, impute or "
|
| 67 |
+
"drop missing values, one-hot encode categoricals, and optionally scale numeric "
|
| 68 |
+
"features. The cleaned dataset is saved temporarily and reused for model training. "
|
| 69 |
+
"Head to the **Data & Preprocessing** tab to run it."
|
| 70 |
+
),
|
| 71 |
+
("missing value", "impute", "nan"): (
|
| 72 |
+
"**Missing values** can be dropped (safe if few) or imputed — mean/median for "
|
| 73 |
+
"numeric, mode for categorical. This app supports both strategies in the "
|
| 74 |
+
"preprocessing step."
|
| 75 |
+
),
|
| 76 |
+
("scaling", "standardize", "normalize"): (
|
| 77 |
+
"**Feature scaling** puts features on comparable ranges. StandardScaler (mean 0, "
|
| 78 |
+
"std 1) matters a lot for SVM and KNN; tree-based models don't need it. Enable it "
|
| 79 |
+
"in the preprocessing options."
|
| 80 |
+
),
|
| 81 |
+
("metric", "accuracy", "precision", "recall", "f1"): (
|
| 82 |
+
"**Classification metrics**: accuracy = overall correct rate; precision = of "
|
| 83 |
+
"predicted positives, how many were right; recall = of actual positives, how many "
|
| 84 |
+
"were found; F1 = harmonic mean of precision and recall (good for imbalanced data)."
|
| 85 |
+
),
|
| 86 |
+
("r2", "r²", "rmse", "mae"): (
|
| 87 |
+
"**Regression metrics**: R² = share of variance explained (1.0 is perfect); "
|
| 88 |
+
"MAE = average absolute error in target units; RMSE = like MAE but penalizes "
|
| 89 |
+
"large errors more."
|
| 90 |
+
),
|
| 91 |
+
("which model", "best model", "what model", "choose a model", "recommend"): (
|
| 92 |
+
"Quick guide: predicting a **number** → start with Linear Regression, then Random "
|
| 93 |
+
"Forest / Gradient Boosting. Predicting a **category** → start with Logistic "
|
| 94 |
+
"Regression, then Random Forest / Gradient Boosting. Small dataset with scaled "
|
| 95 |
+
"features → SVM or KNN are worth a try. Upload your data and I can suggest a task "
|
| 96 |
+
"type automatically from your target column."
|
| 97 |
+
),
|
| 98 |
+
("train test split", "test size", "split"): (
|
| 99 |
+
"**Train/test split** holds out part of the data (default 20% here) to evaluate the "
|
| 100 |
+
"model on unseen examples. Adjust it with the slider in the Model Training tab."
|
| 101 |
+
),
|
| 102 |
+
("feature importance",): (
|
| 103 |
+
"**Feature importance** shows which columns most influence predictions. Tree models "
|
| 104 |
+
"expose `feature_importances_`; linear models use coefficient magnitudes. This app "
|
| 105 |
+
"plots the top 15 automatically after training."
|
| 106 |
+
),
|
| 107 |
+
("cross validation", "cross-validation", "k-fold"): (
|
| 108 |
+
"**Cross-validation** splits data into K folds, trains K times each holding out one "
|
| 109 |
+
"fold, and averages the scores — a more reliable estimate than a single split, at "
|
| 110 |
+
"the cost of K× training time."
|
| 111 |
+
),
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
DATA_LOADED_PROMPT = (
|
| 115 |
+
"\n\n📊 I can see you've uploaded **{name}**. Do you want me to analyse this data? "
|
| 116 |
+
"Head to the **Data & Preprocessing** tab to clean it, then train models in "
|
| 117 |
+
"**Model Training**."
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
FALLBACK = (
|
| 121 |
+
"I can help with questions about the models in this app — linear/logistic regression, "
|
| 122 |
+
"decision trees, random forests, gradient boosting, SVM, KNN — plus preprocessing, "
|
| 123 |
+
"metrics, overfitting, and train/test splits.\n\n"
|
| 124 |
+
"Or just get started: **upload your data in the Data & Preprocessing tab** and I'll "
|
| 125 |
+
"help you analyse it. What do you want to analyse today?"
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
GREETINGS = ("hi", "hello", "hey", "good morning", "good afternoon", "good evening", "yo", "hola")
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def respond(message: str, uploaded_file_name: str | None = None) -> str:
|
| 132 |
+
"""Return the assistant's reply to a user message."""
|
| 133 |
+
msg = message.lower().strip()
|
| 134 |
+
|
| 135 |
+
if any(msg == g or msg.startswith(g + " ") or msg.startswith(g + "!") for g in GREETINGS):
|
| 136 |
+
reply = (
|
| 137 |
+
"Hello! I'm here to help you analyse your data with machine learning. "
|
| 138 |
+
"**What do you want to analyse today?** You can ask about any ML model, "
|
| 139 |
+
"or upload a dataset (CSV, Excel, JSON, Parquet) in the "
|
| 140 |
+
"**Data & Preprocessing** tab."
|
| 141 |
+
)
|
| 142 |
+
elif any(kw in msg for kw in ("analyse", "analyze", "analysis", "get started", "start", "upload")):
|
| 143 |
+
reply = (
|
| 144 |
+
"Great — here's the workflow:\n\n"
|
| 145 |
+
"1. **Data & Preprocessing tab** — upload your file, review the profile, and "
|
| 146 |
+
"run cleaning (missing values, duplicates, encoding, scaling).\n"
|
| 147 |
+
"2. **Model Training tab** — pick a target column, a task "
|
| 148 |
+
"(regression/classification), and a model, then train.\n"
|
| 149 |
+
"3. Review metrics and charts, open the **code dropdown** to see the exact "
|
| 150 |
+
"Python used, and **download the report** (PDF/HTML) and cleaned data.\n\n"
|
| 151 |
+
"Do you want me to analyse your data? Upload it and I'll take it from there!"
|
| 152 |
+
)
|
| 153 |
+
else:
|
| 154 |
+
reply = None
|
| 155 |
+
for keywords, answer in KNOWLEDGE_BASE.items():
|
| 156 |
+
if any(kw in msg for kw in keywords):
|
| 157 |
+
reply = answer
|
| 158 |
+
break
|
| 159 |
+
if reply is None:
|
| 160 |
+
reply = FALLBACK
|
| 161 |
+
|
| 162 |
+
if uploaded_file_name:
|
| 163 |
+
reply += DATA_LOADED_PROMPT.format(name=uploaded_file_name)
|
| 164 |
+
return reply
|
data_processor.py
ADDED
|
@@ -0,0 +1,212 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Data loading, profiling, and preprocessing utilities."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
import tempfile
|
| 5 |
+
|
| 6 |
+
import numpy as np
|
| 7 |
+
import pandas as pd
|
| 8 |
+
|
| 9 |
+
# Temp workspace that persists for the app session
|
| 10 |
+
TEMP_DIR = os.path.join(tempfile.gettempdir(), "ml_analysis_space")
|
| 11 |
+
os.makedirs(TEMP_DIR, exist_ok=True)
|
| 12 |
+
|
| 13 |
+
CLEANED_PATH = os.path.join(TEMP_DIR, "cleaned_data.csv")
|
| 14 |
+
|
| 15 |
+
SUPPORTED_EXTENSIONS = [".csv", ".tsv", ".xlsx", ".xls", ".json", ".parquet", ".txt"]
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def load_data(file_path: str) -> pd.DataFrame:
|
| 19 |
+
"""Load a dataset from CSV, TSV, Excel, JSON, or Parquet."""
|
| 20 |
+
ext = os.path.splitext(file_path)[1].lower()
|
| 21 |
+
if ext in (".csv", ".txt"):
|
| 22 |
+
df = pd.read_csv(file_path)
|
| 23 |
+
elif ext == ".tsv":
|
| 24 |
+
df = pd.read_csv(file_path, sep="\t")
|
| 25 |
+
elif ext in (".xlsx", ".xls"):
|
| 26 |
+
df = pd.read_excel(file_path)
|
| 27 |
+
elif ext == ".json":
|
| 28 |
+
df = pd.read_json(file_path)
|
| 29 |
+
elif ext == ".parquet":
|
| 30 |
+
df = pd.read_parquet(file_path)
|
| 31 |
+
else:
|
| 32 |
+
raise ValueError(
|
| 33 |
+
f"Unsupported file type '{ext}'. Supported: {', '.join(SUPPORTED_EXTENSIONS)}"
|
| 34 |
+
)
|
| 35 |
+
return df
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def profile_data(df: pd.DataFrame) -> dict:
|
| 39 |
+
"""Return a summary profile of the dataset."""
|
| 40 |
+
numeric_cols = df.select_dtypes(include=np.number).columns.tolist()
|
| 41 |
+
categorical_cols = df.select_dtypes(exclude=np.number).columns.tolist()
|
| 42 |
+
profile = {
|
| 43 |
+
"n_rows": int(df.shape[0]),
|
| 44 |
+
"n_cols": int(df.shape[1]),
|
| 45 |
+
"columns": df.columns.tolist(),
|
| 46 |
+
"dtypes": {c: str(t) for c, t in df.dtypes.items()},
|
| 47 |
+
"numeric_columns": numeric_cols,
|
| 48 |
+
"categorical_columns": categorical_cols,
|
| 49 |
+
"missing_counts": df.isna().sum().to_dict(),
|
| 50 |
+
"missing_total": int(df.isna().sum().sum()),
|
| 51 |
+
"duplicate_rows": int(df.duplicated().sum()),
|
| 52 |
+
"memory_kb": round(df.memory_usage(deep=True).sum() / 1024, 1),
|
| 53 |
+
}
|
| 54 |
+
return profile
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def profile_text(profile: dict) -> str:
|
| 58 |
+
"""Human-readable summary of a profile dict."""
|
| 59 |
+
lines = [
|
| 60 |
+
f"**Rows:** {profile['n_rows']:,} | **Columns:** {profile['n_cols']} "
|
| 61 |
+
f"| **Memory:** {profile['memory_kb']} KB",
|
| 62 |
+
f"**Numeric columns ({len(profile['numeric_columns'])}):** "
|
| 63 |
+
+ (", ".join(profile["numeric_columns"]) or "none"),
|
| 64 |
+
f"**Categorical columns ({len(profile['categorical_columns'])}):** "
|
| 65 |
+
+ (", ".join(profile["categorical_columns"]) or "none"),
|
| 66 |
+
f"**Missing values:** {profile['missing_total']:,} | "
|
| 67 |
+
f"**Duplicate rows:** {profile['duplicate_rows']:,}",
|
| 68 |
+
]
|
| 69 |
+
missing = {k: v for k, v in profile["missing_counts"].items() if v > 0}
|
| 70 |
+
if missing:
|
| 71 |
+
lines.append(
|
| 72 |
+
"**Columns with missing values:** "
|
| 73 |
+
+ ", ".join(f"{k} ({v})" for k, v in missing.items())
|
| 74 |
+
)
|
| 75 |
+
return "\n\n".join(lines)
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def preprocess_data(
|
| 79 |
+
df: pd.DataFrame,
|
| 80 |
+
missing_strategy: str = "Impute (mean/mode)",
|
| 81 |
+
drop_duplicates: bool = True,
|
| 82 |
+
encode_categoricals: bool = True,
|
| 83 |
+
scale_numeric: bool = False,
|
| 84 |
+
target_column: str | None = None,
|
| 85 |
+
) -> tuple[pd.DataFrame, list[str]]:
|
| 86 |
+
"""Clean the dataset and return (cleaned_df, list of steps applied)."""
|
| 87 |
+
steps = []
|
| 88 |
+
df = df.copy()
|
| 89 |
+
|
| 90 |
+
# Drop columns that are entirely empty
|
| 91 |
+
empty_cols = [c for c in df.columns if df[c].isna().all()]
|
| 92 |
+
if empty_cols:
|
| 93 |
+
df = df.drop(columns=empty_cols)
|
| 94 |
+
steps.append(f"Dropped fully-empty columns: {', '.join(empty_cols)}")
|
| 95 |
+
|
| 96 |
+
# Duplicates
|
| 97 |
+
if drop_duplicates:
|
| 98 |
+
n = int(df.duplicated().sum())
|
| 99 |
+
if n:
|
| 100 |
+
df = df.drop_duplicates().reset_index(drop=True)
|
| 101 |
+
steps.append(f"Removed {n} duplicate rows")
|
| 102 |
+
|
| 103 |
+
# Missing values
|
| 104 |
+
n_missing = int(df.isna().sum().sum())
|
| 105 |
+
if n_missing:
|
| 106 |
+
if missing_strategy.startswith("Drop"):
|
| 107 |
+
before = len(df)
|
| 108 |
+
df = df.dropna().reset_index(drop=True)
|
| 109 |
+
steps.append(f"Dropped {before - len(df)} rows with missing values")
|
| 110 |
+
else:
|
| 111 |
+
for col in df.columns:
|
| 112 |
+
if df[col].isna().any():
|
| 113 |
+
if pd.api.types.is_numeric_dtype(df[col]):
|
| 114 |
+
df[col] = df[col].fillna(df[col].mean())
|
| 115 |
+
else:
|
| 116 |
+
mode = df[col].mode()
|
| 117 |
+
df[col] = df[col].fillna(mode.iloc[0] if len(mode) else "unknown")
|
| 118 |
+
steps.append(
|
| 119 |
+
f"Imputed {n_missing} missing values (mean for numeric, mode for categorical)"
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
# Encode categoricals (except the target, which models handle separately)
|
| 123 |
+
if encode_categoricals:
|
| 124 |
+
cat_cols = [
|
| 125 |
+
c
|
| 126 |
+
for c in df.select_dtypes(exclude=np.number).columns
|
| 127 |
+
if c != target_column
|
| 128 |
+
]
|
| 129 |
+
low_card = [c for c in cat_cols if df[c].nunique() <= 20]
|
| 130 |
+
high_card = [c for c in cat_cols if df[c].nunique() > 20]
|
| 131 |
+
if high_card:
|
| 132 |
+
df = df.drop(columns=high_card)
|
| 133 |
+
steps.append(
|
| 134 |
+
f"Dropped high-cardinality text columns (>20 unique): {', '.join(high_card)}"
|
| 135 |
+
)
|
| 136 |
+
if low_card:
|
| 137 |
+
df = pd.get_dummies(df, columns=low_card, drop_first=True, dtype=int)
|
| 138 |
+
steps.append(f"One-hot encoded: {', '.join(low_card)}")
|
| 139 |
+
|
| 140 |
+
# Scale numeric features
|
| 141 |
+
if scale_numeric:
|
| 142 |
+
from sklearn.preprocessing import StandardScaler
|
| 143 |
+
|
| 144 |
+
num_cols = [
|
| 145 |
+
c
|
| 146 |
+
for c in df.select_dtypes(include=np.number).columns
|
| 147 |
+
if c != target_column
|
| 148 |
+
]
|
| 149 |
+
if num_cols:
|
| 150 |
+
df[num_cols] = StandardScaler().fit_transform(df[num_cols])
|
| 151 |
+
steps.append(f"Standard-scaled {len(num_cols)} numeric columns")
|
| 152 |
+
|
| 153 |
+
if not steps:
|
| 154 |
+
steps.append("Data was already clean — no changes needed")
|
| 155 |
+
|
| 156 |
+
df.to_csv(CLEANED_PATH, index=False)
|
| 157 |
+
steps.append(f"Saved cleaned data ({len(df):,} rows x {df.shape[1]} cols) for modeling")
|
| 158 |
+
return df, steps
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def generate_preprocessing_code(
|
| 162 |
+
missing_strategy: str,
|
| 163 |
+
drop_duplicates: bool,
|
| 164 |
+
encode_categoricals: bool,
|
| 165 |
+
scale_numeric: bool,
|
| 166 |
+
target_column: str | None,
|
| 167 |
+
) -> str:
|
| 168 |
+
"""Return equivalent standalone pandas code for the preprocessing performed."""
|
| 169 |
+
code = [
|
| 170 |
+
"import pandas as pd",
|
| 171 |
+
"import numpy as np",
|
| 172 |
+
"",
|
| 173 |
+
"df = pd.read_csv('your_data.csv')",
|
| 174 |
+
"",
|
| 175 |
+
"# Drop fully-empty columns",
|
| 176 |
+
"df = df.dropna(axis=1, how='all')",
|
| 177 |
+
]
|
| 178 |
+
if drop_duplicates:
|
| 179 |
+
code += ["", "# Remove duplicate rows", "df = df.drop_duplicates().reset_index(drop=True)"]
|
| 180 |
+
if missing_strategy.startswith("Drop"):
|
| 181 |
+
code += ["", "# Drop rows with missing values", "df = df.dropna().reset_index(drop=True)"]
|
| 182 |
+
else:
|
| 183 |
+
code += [
|
| 184 |
+
"",
|
| 185 |
+
"# Impute missing values: mean for numeric, mode for categorical",
|
| 186 |
+
"for col in df.columns:",
|
| 187 |
+
" if df[col].isna().any():",
|
| 188 |
+
" if pd.api.types.is_numeric_dtype(df[col]):",
|
| 189 |
+
" df[col] = df[col].fillna(df[col].mean())",
|
| 190 |
+
" else:",
|
| 191 |
+
" df[col] = df[col].fillna(df[col].mode().iloc[0])",
|
| 192 |
+
]
|
| 193 |
+
if encode_categoricals:
|
| 194 |
+
code += [
|
| 195 |
+
"",
|
| 196 |
+
"# One-hot encode low-cardinality categoricals (drop high-cardinality text)",
|
| 197 |
+
f"target = {target_column!r}",
|
| 198 |
+
"cat_cols = [c for c in df.select_dtypes(exclude=np.number).columns if c != target]",
|
| 199 |
+
"df = df.drop(columns=[c for c in cat_cols if df[c].nunique() > 20])",
|
| 200 |
+
"cat_cols = [c for c in df.select_dtypes(exclude=np.number).columns if c != target]",
|
| 201 |
+
"df = pd.get_dummies(df, columns=cat_cols, drop_first=True, dtype=int)",
|
| 202 |
+
]
|
| 203 |
+
if scale_numeric:
|
| 204 |
+
code += [
|
| 205 |
+
"",
|
| 206 |
+
"# Standard-scale numeric features",
|
| 207 |
+
"from sklearn.preprocessing import StandardScaler",
|
| 208 |
+
f"num_cols = [c for c in df.select_dtypes(include=np.number).columns if c != {target_column!r}]",
|
| 209 |
+
"df[num_cols] = StandardScaler().fit_transform(df[num_cols])",
|
| 210 |
+
]
|
| 211 |
+
code += ["", "df.to_csv('cleaned_data.csv', index=False)"]
|
| 212 |
+
return "\n".join(code)
|
ml_models.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Model training, evaluation, and code generation."""
|
| 2 |
+
|
| 3 |
+
import io
|
| 4 |
+
import os
|
| 5 |
+
|
| 6 |
+
import matplotlib
|
| 7 |
+
|
| 8 |
+
matplotlib.use("Agg")
|
| 9 |
+
import matplotlib.pyplot as plt
|
| 10 |
+
import numpy as np
|
| 11 |
+
import pandas as pd
|
| 12 |
+
from sklearn.ensemble import (
|
| 13 |
+
GradientBoostingClassifier,
|
| 14 |
+
GradientBoostingRegressor,
|
| 15 |
+
RandomForestClassifier,
|
| 16 |
+
RandomForestRegressor,
|
| 17 |
+
)
|
| 18 |
+
from sklearn.linear_model import LinearRegression, LogisticRegression
|
| 19 |
+
from sklearn.metrics import (
|
| 20 |
+
accuracy_score,
|
| 21 |
+
confusion_matrix,
|
| 22 |
+
f1_score,
|
| 23 |
+
mean_absolute_error,
|
| 24 |
+
mean_squared_error,
|
| 25 |
+
precision_score,
|
| 26 |
+
r2_score,
|
| 27 |
+
recall_score,
|
| 28 |
+
)
|
| 29 |
+
from sklearn.model_selection import train_test_split
|
| 30 |
+
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
|
| 31 |
+
from sklearn.preprocessing import LabelEncoder
|
| 32 |
+
from sklearn.svm import SVC, SVR
|
| 33 |
+
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
| 34 |
+
|
| 35 |
+
from data_processor import TEMP_DIR
|
| 36 |
+
|
| 37 |
+
REGRESSION_MODELS = {
|
| 38 |
+
"Linear Regression": LinearRegression,
|
| 39 |
+
"Decision Tree Regressor": DecisionTreeRegressor,
|
| 40 |
+
"Random Forest Regressor": RandomForestRegressor,
|
| 41 |
+
"Gradient Boosting Regressor": GradientBoostingRegressor,
|
| 42 |
+
"Support Vector Regressor (SVR)": SVR,
|
| 43 |
+
"K-Nearest Neighbors Regressor": KNeighborsRegressor,
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
CLASSIFICATION_MODELS = {
|
| 47 |
+
"Logistic Regression": LogisticRegression,
|
| 48 |
+
"Decision Tree Classifier": DecisionTreeClassifier,
|
| 49 |
+
"Random Forest Classifier": RandomForestClassifier,
|
| 50 |
+
"Gradient Boosting Classifier": GradientBoostingClassifier,
|
| 51 |
+
"Support Vector Classifier (SVC)": SVC,
|
| 52 |
+
"K-Nearest Neighbors Classifier": KNeighborsClassifier,
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
MODEL_IMPORTS = {
|
| 56 |
+
"Linear Regression": "from sklearn.linear_model import LinearRegression",
|
| 57 |
+
"Decision Tree Regressor": "from sklearn.tree import DecisionTreeRegressor",
|
| 58 |
+
"Random Forest Regressor": "from sklearn.ensemble import RandomForestRegressor",
|
| 59 |
+
"Gradient Boosting Regressor": "from sklearn.ensemble import GradientBoostingRegressor",
|
| 60 |
+
"Support Vector Regressor (SVR)": "from sklearn.svm import SVR",
|
| 61 |
+
"K-Nearest Neighbors Regressor": "from sklearn.neighbors import KNeighborsRegressor",
|
| 62 |
+
"Logistic Regression": "from sklearn.linear_model import LogisticRegression",
|
| 63 |
+
"Decision Tree Classifier": "from sklearn.tree import DecisionTreeClassifier",
|
| 64 |
+
"Random Forest Classifier": "from sklearn.ensemble import RandomForestClassifier",
|
| 65 |
+
"Gradient Boosting Classifier": "from sklearn.ensemble import GradientBoostingClassifier",
|
| 66 |
+
"Support Vector Classifier (SVC)": "from sklearn.svm import SVC",
|
| 67 |
+
"K-Nearest Neighbors Classifier": "from sklearn.neighbors import KNeighborsClassifier",
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
def suggest_task(df: pd.DataFrame, target: str) -> str:
|
| 72 |
+
"""Suggest 'Classification' or 'Regression' based on the target column."""
|
| 73 |
+
s = df[target]
|
| 74 |
+
if not pd.api.types.is_numeric_dtype(s):
|
| 75 |
+
return "Classification"
|
| 76 |
+
if s.nunique() <= 10:
|
| 77 |
+
return "Classification"
|
| 78 |
+
return "Regression"
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _make_model(name: str, task: str):
|
| 82 |
+
cls = (CLASSIFICATION_MODELS if task == "Classification" else REGRESSION_MODELS)[name]
|
| 83 |
+
kwargs = {}
|
| 84 |
+
if "Logistic" in name:
|
| 85 |
+
kwargs["max_iter"] = 1000
|
| 86 |
+
if "Random Forest" in name or "Gradient Boosting" in name:
|
| 87 |
+
kwargs["random_state"] = 42
|
| 88 |
+
if "Decision Tree" in name:
|
| 89 |
+
kwargs["random_state"] = 42
|
| 90 |
+
return cls(**kwargs)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def train_model(
|
| 94 |
+
df: pd.DataFrame,
|
| 95 |
+
target: str,
|
| 96 |
+
model_name: str,
|
| 97 |
+
task: str,
|
| 98 |
+
test_size: float = 0.2,
|
| 99 |
+
) -> dict:
|
| 100 |
+
"""Train a model and return metrics, plot path, and predictions sample."""
|
| 101 |
+
df = df.dropna(subset=[target])
|
| 102 |
+
X = df.drop(columns=[target]).select_dtypes(include=np.number)
|
| 103 |
+
if X.empty:
|
| 104 |
+
raise ValueError("No numeric feature columns available. Run preprocessing first.")
|
| 105 |
+
y = df[target]
|
| 106 |
+
|
| 107 |
+
label_encoder = None
|
| 108 |
+
if task == "Classification" and not pd.api.types.is_numeric_dtype(y):
|
| 109 |
+
label_encoder = LabelEncoder()
|
| 110 |
+
y = pd.Series(label_encoder.fit_transform(y), index=df.index)
|
| 111 |
+
|
| 112 |
+
stratify = y if (task == "Classification" and y.value_counts().min() >= 2) else None
|
| 113 |
+
X_train, X_test, y_train, y_test = train_test_split(
|
| 114 |
+
X, y, test_size=test_size, random_state=42, stratify=stratify
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
model = _make_model(model_name, task)
|
| 118 |
+
model.fit(X_train, y_train)
|
| 119 |
+
y_pred = model.predict(X_test)
|
| 120 |
+
|
| 121 |
+
if task == "Classification":
|
| 122 |
+
avg = "binary" if pd.Series(y).nunique() == 2 else "weighted"
|
| 123 |
+
metrics = {
|
| 124 |
+
"Accuracy": round(accuracy_score(y_test, y_pred), 4),
|
| 125 |
+
"Precision": round(precision_score(y_test, y_pred, average=avg, zero_division=0), 4),
|
| 126 |
+
"Recall": round(recall_score(y_test, y_pred, average=avg, zero_division=0), 4),
|
| 127 |
+
"F1 Score": round(f1_score(y_test, y_pred, average=avg, zero_division=0), 4),
|
| 128 |
+
}
|
| 129 |
+
plot_path = _plot_confusion(y_test, y_pred, model_name, label_encoder)
|
| 130 |
+
else:
|
| 131 |
+
metrics = {
|
| 132 |
+
"R² Score": round(r2_score(y_test, y_pred), 4),
|
| 133 |
+
"MAE": round(mean_absolute_error(y_test, y_pred), 4),
|
| 134 |
+
"RMSE": round(float(np.sqrt(mean_squared_error(y_test, y_pred))), 4),
|
| 135 |
+
}
|
| 136 |
+
plot_path = _plot_regression(y_test, y_pred, model_name)
|
| 137 |
+
|
| 138 |
+
importance_path = _plot_importance(model, X.columns, model_name)
|
| 139 |
+
|
| 140 |
+
return {
|
| 141 |
+
"model_name": model_name,
|
| 142 |
+
"task": task,
|
| 143 |
+
"target": target,
|
| 144 |
+
"features": X.columns.tolist(),
|
| 145 |
+
"n_train": len(X_train),
|
| 146 |
+
"n_test": len(X_test),
|
| 147 |
+
"metrics": metrics,
|
| 148 |
+
"plot_path": plot_path,
|
| 149 |
+
"importance_path": importance_path,
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
def _plot_confusion(y_test, y_pred, model_name, label_encoder=None):
|
| 154 |
+
cm = confusion_matrix(y_test, y_pred)
|
| 155 |
+
labels = label_encoder.classes_ if label_encoder is not None else sorted(set(y_test))
|
| 156 |
+
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
| 157 |
+
im = ax.imshow(cm, cmap="Blues")
|
| 158 |
+
ax.set_xticks(range(len(labels)), labels=[str(l) for l in labels], rotation=45, ha="right")
|
| 159 |
+
ax.set_yticks(range(len(labels)), labels=[str(l) for l in labels])
|
| 160 |
+
for i in range(cm.shape[0]):
|
| 161 |
+
for j in range(cm.shape[1]):
|
| 162 |
+
ax.text(j, i, cm[i, j], ha="center", va="center",
|
| 163 |
+
color="white" if cm[i, j] > cm.max() / 2 else "black")
|
| 164 |
+
ax.set_xlabel("Predicted")
|
| 165 |
+
ax.set_ylabel("Actual")
|
| 166 |
+
ax.set_title(f"Confusion Matrix — {model_name}")
|
| 167 |
+
fig.colorbar(im)
|
| 168 |
+
fig.tight_layout()
|
| 169 |
+
path = os.path.join(TEMP_DIR, "result_plot.png")
|
| 170 |
+
fig.savefig(path, dpi=120)
|
| 171 |
+
plt.close(fig)
|
| 172 |
+
return path
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _plot_regression(y_test, y_pred, model_name):
|
| 176 |
+
fig, ax = plt.subplots(figsize=(5.5, 4.5))
|
| 177 |
+
ax.scatter(y_test, y_pred, alpha=0.5, edgecolors="none")
|
| 178 |
+
lims = [min(y_test.min(), y_pred.min()), max(y_test.max(), y_pred.max())]
|
| 179 |
+
ax.plot(lims, lims, "r--", label="Perfect prediction")
|
| 180 |
+
ax.set_xlabel("Actual")
|
| 181 |
+
ax.set_ylabel("Predicted")
|
| 182 |
+
ax.set_title(f"Actual vs Predicted — {model_name}")
|
| 183 |
+
ax.legend()
|
| 184 |
+
fig.tight_layout()
|
| 185 |
+
path = os.path.join(TEMP_DIR, "result_plot.png")
|
| 186 |
+
fig.savefig(path, dpi=120)
|
| 187 |
+
plt.close(fig)
|
| 188 |
+
return path
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def _plot_importance(model, feature_names, model_name):
|
| 192 |
+
importances = None
|
| 193 |
+
if hasattr(model, "feature_importances_"):
|
| 194 |
+
importances = model.feature_importances_
|
| 195 |
+
elif hasattr(model, "coef_"):
|
| 196 |
+
coef = model.coef_
|
| 197 |
+
importances = np.abs(coef).mean(axis=0) if coef.ndim > 1 else np.abs(coef)
|
| 198 |
+
if importances is None:
|
| 199 |
+
return None
|
| 200 |
+
order = np.argsort(importances)[-15:]
|
| 201 |
+
fig, ax = plt.subplots(figsize=(6, 4.5))
|
| 202 |
+
ax.barh([feature_names[i] for i in order], importances[order], color="#4C72B0")
|
| 203 |
+
ax.set_title(f"Feature Importance — {model_name}")
|
| 204 |
+
fig.tight_layout()
|
| 205 |
+
path = os.path.join(TEMP_DIR, "importance_plot.png")
|
| 206 |
+
fig.savefig(path, dpi=120)
|
| 207 |
+
plt.close(fig)
|
| 208 |
+
return path
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def generate_model_code(model_name: str, task: str, target: str, test_size: float) -> str:
|
| 212 |
+
"""Return equivalent standalone sklearn code."""
|
| 213 |
+
cls = (CLASSIFICATION_MODELS if task == "Classification" else REGRESSION_MODELS)[model_name]
|
| 214 |
+
kwargs = []
|
| 215 |
+
if "Logistic" in model_name:
|
| 216 |
+
kwargs.append("max_iter=1000")
|
| 217 |
+
if any(k in model_name for k in ("Random Forest", "Gradient Boosting", "Decision Tree")):
|
| 218 |
+
kwargs.append("random_state=42")
|
| 219 |
+
ctor = f"{cls.__name__}({', '.join(kwargs)})"
|
| 220 |
+
|
| 221 |
+
lines = [
|
| 222 |
+
"import numpy as np",
|
| 223 |
+
"import pandas as pd",
|
| 224 |
+
"from sklearn.model_selection import train_test_split",
|
| 225 |
+
MODEL_IMPORTS[model_name],
|
| 226 |
+
]
|
| 227 |
+
if task == "Classification":
|
| 228 |
+
lines += [
|
| 229 |
+
"from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score",
|
| 230 |
+
"from sklearn.preprocessing import LabelEncoder",
|
| 231 |
+
]
|
| 232 |
+
else:
|
| 233 |
+
lines.append(
|
| 234 |
+
"from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error"
|
| 235 |
+
)
|
| 236 |
+
lines += [
|
| 237 |
+
"",
|
| 238 |
+
"df = pd.read_csv('cleaned_data.csv')",
|
| 239 |
+
f"target = {target!r}",
|
| 240 |
+
"X = df.drop(columns=[target]).select_dtypes(include=np.number)",
|
| 241 |
+
"y = df[target]",
|
| 242 |
+
]
|
| 243 |
+
if task == "Classification":
|
| 244 |
+
lines += [
|
| 245 |
+
"if not pd.api.types.is_numeric_dtype(y):",
|
| 246 |
+
" y = LabelEncoder().fit_transform(y)",
|
| 247 |
+
]
|
| 248 |
+
lines += [
|
| 249 |
+
"",
|
| 250 |
+
"X_train, X_test, y_train, y_test = train_test_split(",
|
| 251 |
+
f" X, y, test_size={test_size}, random_state=42)",
|
| 252 |
+
"",
|
| 253 |
+
f"model = {ctor}",
|
| 254 |
+
"model.fit(X_train, y_train)",
|
| 255 |
+
"y_pred = model.predict(X_test)",
|
| 256 |
+
"",
|
| 257 |
+
]
|
| 258 |
+
if task == "Classification":
|
| 259 |
+
lines += [
|
| 260 |
+
"print('Accuracy :', accuracy_score(y_test, y_pred))",
|
| 261 |
+
"print('Precision:', precision_score(y_test, y_pred, average='weighted'))",
|
| 262 |
+
"print('Recall :', recall_score(y_test, y_pred, average='weighted'))",
|
| 263 |
+
"print('F1 Score :', f1_score(y_test, y_pred, average='weighted'))",
|
| 264 |
+
]
|
| 265 |
+
else:
|
| 266 |
+
lines += [
|
| 267 |
+
"print('R2 :', r2_score(y_test, y_pred))",
|
| 268 |
+
"print('MAE :', mean_absolute_error(y_test, y_pred))",
|
| 269 |
+
"print('RMSE:', np.sqrt(mean_squared_error(y_test, y_pred)))",
|
| 270 |
+
]
|
| 271 |
+
return "\n".join(lines)
|
report_generator.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Generate downloadable PDF and HTML analysis reports."""
|
| 2 |
+
|
| 3 |
+
import base64
|
| 4 |
+
import os
|
| 5 |
+
from datetime import datetime
|
| 6 |
+
|
| 7 |
+
from fpdf import FPDF
|
| 8 |
+
|
| 9 |
+
from data_processor import TEMP_DIR
|
| 10 |
+
|
| 11 |
+
PDF_PATH = os.path.join(TEMP_DIR, "analysis_report.pdf")
|
| 12 |
+
HTML_PATH = os.path.join(TEMP_DIR, "analysis_report.html")
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _sanitize(text: str) -> str:
|
| 16 |
+
"""fpdf core fonts are latin-1 only."""
|
| 17 |
+
replacements = {"²": "^2", "—": "-", "–": "-", "'": "'", "'": "'", """: '"', """: '"', "×": "x"}
|
| 18 |
+
for a, b in replacements.items():
|
| 19 |
+
text = text.replace(a, b)
|
| 20 |
+
return text.encode("latin-1", "replace").decode("latin-1")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def generate_pdf_report(profile: dict, steps: list[str], result: dict) -> str:
|
| 24 |
+
pdf = FPDF()
|
| 25 |
+
pdf.set_auto_page_break(auto=True, margin=15)
|
| 26 |
+
pdf.add_page()
|
| 27 |
+
|
| 28 |
+
pdf.set_font("Helvetica", "B", 18)
|
| 29 |
+
pdf.cell(0, 12, "ML Data Analysis Report", ln=True)
|
| 30 |
+
pdf.set_font("Helvetica", "", 10)
|
| 31 |
+
pdf.set_text_color(120, 120, 120)
|
| 32 |
+
pdf.cell(0, 6, f"Generated {datetime.now():%Y-%m-%d %H:%M}", ln=True)
|
| 33 |
+
pdf.set_text_color(0, 0, 0)
|
| 34 |
+
pdf.ln(4)
|
| 35 |
+
|
| 36 |
+
def section(title):
|
| 37 |
+
pdf.set_font("Helvetica", "B", 13)
|
| 38 |
+
pdf.set_fill_color(230, 236, 245)
|
| 39 |
+
pdf.cell(0, 9, _sanitize(title), ln=True, fill=True)
|
| 40 |
+
pdf.ln(2)
|
| 41 |
+
pdf.set_font("Helvetica", "", 10)
|
| 42 |
+
|
| 43 |
+
def kv(key, value):
|
| 44 |
+
pdf.set_font("Helvetica", "B", 10)
|
| 45 |
+
pdf.cell(55, 6, _sanitize(str(key)))
|
| 46 |
+
pdf.set_font("Helvetica", "", 10)
|
| 47 |
+
pdf.multi_cell(0, 6, _sanitize(str(value)))
|
| 48 |
+
|
| 49 |
+
section("1. Dataset Overview")
|
| 50 |
+
kv("Rows", f"{profile['n_rows']:,}")
|
| 51 |
+
kv("Columns", profile["n_cols"])
|
| 52 |
+
kv("Numeric columns", ", ".join(profile["numeric_columns"]) or "none")
|
| 53 |
+
kv("Categorical columns", ", ".join(profile["categorical_columns"]) or "none")
|
| 54 |
+
kv("Missing values", f"{profile['missing_total']:,}")
|
| 55 |
+
kv("Duplicate rows", f"{profile['duplicate_rows']:,}")
|
| 56 |
+
pdf.ln(4)
|
| 57 |
+
|
| 58 |
+
section("2. Preprocessing Steps")
|
| 59 |
+
for i, step in enumerate(steps, 1):
|
| 60 |
+
pdf.multi_cell(0, 6, _sanitize(f"{i}. {step}"))
|
| 61 |
+
pdf.ln(4)
|
| 62 |
+
|
| 63 |
+
section("3. Model & Results")
|
| 64 |
+
kv("Model", result["model_name"])
|
| 65 |
+
kv("Task", result["task"])
|
| 66 |
+
kv("Target column", result["target"])
|
| 67 |
+
kv("Training samples", f"{result['n_train']:,}")
|
| 68 |
+
kv("Test samples", f"{result['n_test']:,}")
|
| 69 |
+
pdf.ln(2)
|
| 70 |
+
for metric, value in result["metrics"].items():
|
| 71 |
+
kv(metric, value)
|
| 72 |
+
pdf.ln(4)
|
| 73 |
+
|
| 74 |
+
for title, path in [
|
| 75 |
+
("4. Result Plot", result.get("plot_path")),
|
| 76 |
+
("5. Feature Importance", result.get("importance_path")),
|
| 77 |
+
]:
|
| 78 |
+
if path and os.path.exists(path):
|
| 79 |
+
section(title)
|
| 80 |
+
pdf.image(path, w=150)
|
| 81 |
+
pdf.ln(4)
|
| 82 |
+
|
| 83 |
+
pdf.output(PDF_PATH)
|
| 84 |
+
return PDF_PATH
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def _img_b64(path):
|
| 88 |
+
if not path or not os.path.exists(path):
|
| 89 |
+
return None
|
| 90 |
+
with open(path, "rb") as f:
|
| 91 |
+
return base64.b64encode(f.read()).decode()
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def generate_html_report(profile: dict, steps: list[str], result: dict) -> str:
|
| 95 |
+
plot_b64 = _img_b64(result.get("plot_path"))
|
| 96 |
+
imp_b64 = _img_b64(result.get("importance_path"))
|
| 97 |
+
|
| 98 |
+
metrics_rows = "".join(
|
| 99 |
+
f"<tr><td>{k}</td><td><b>{v}</b></td></tr>" for k, v in result["metrics"].items()
|
| 100 |
+
)
|
| 101 |
+
steps_html = "".join(f"<li>{s}</li>" for s in steps)
|
| 102 |
+
missing = {k: v for k, v in profile["missing_counts"].items() if v > 0}
|
| 103 |
+
missing_html = (
|
| 104 |
+
", ".join(f"{k} ({v})" for k, v in missing.items()) if missing else "None"
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
html = f"""<!DOCTYPE html>
|
| 108 |
+
<html><head><meta charset="utf-8"><title>ML Data Analysis Report</title>
|
| 109 |
+
<style>
|
| 110 |
+
body {{ font-family: -apple-system, 'Segoe UI', Roboto, sans-serif; max-width: 860px;
|
| 111 |
+
margin: 40px auto; padding: 0 20px; color: #1a202c; line-height: 1.6; }}
|
| 112 |
+
h1 {{ border-bottom: 3px solid #4C72B0; padding-bottom: 8px; }}
|
| 113 |
+
h2 {{ color: #4C72B0; margin-top: 32px; }}
|
| 114 |
+
table {{ border-collapse: collapse; width: 100%; margin: 12px 0; }}
|
| 115 |
+
td, th {{ border: 1px solid #e2e8f0; padding: 8px 12px; text-align: left; }}
|
| 116 |
+
th {{ background: #edf2f7; }}
|
| 117 |
+
img {{ max-width: 100%; border: 1px solid #e2e8f0; border-radius: 8px; margin: 8px 0; }}
|
| 118 |
+
.meta {{ color: #718096; font-size: 0.9em; }}
|
| 119 |
+
</style></head><body>
|
| 120 |
+
<h1>ML Data Analysis Report</h1>
|
| 121 |
+
<p class="meta">Generated {datetime.now():%Y-%m-%d %H:%M}</p>
|
| 122 |
+
|
| 123 |
+
<h2>1. Dataset Overview</h2>
|
| 124 |
+
<table>
|
| 125 |
+
<tr><th>Rows</th><td>{profile['n_rows']:,}</td></tr>
|
| 126 |
+
<tr><th>Columns</th><td>{profile['n_cols']}</td></tr>
|
| 127 |
+
<tr><th>Numeric columns</th><td>{', '.join(profile['numeric_columns']) or 'none'}</td></tr>
|
| 128 |
+
<tr><th>Categorical columns</th><td>{', '.join(profile['categorical_columns']) or 'none'}</td></tr>
|
| 129 |
+
<tr><th>Missing values</th><td>{profile['missing_total']:,} ({missing_html})</td></tr>
|
| 130 |
+
<tr><th>Duplicate rows</th><td>{profile['duplicate_rows']:,}</td></tr>
|
| 131 |
+
</table>
|
| 132 |
+
|
| 133 |
+
<h2>2. Preprocessing Steps</h2>
|
| 134 |
+
<ol>{steps_html}</ol>
|
| 135 |
+
|
| 136 |
+
<h2>3. Model & Results</h2>
|
| 137 |
+
<table>
|
| 138 |
+
<tr><th>Model</th><td>{result['model_name']}</td></tr>
|
| 139 |
+
<tr><th>Task</th><td>{result['task']}</td></tr>
|
| 140 |
+
<tr><th>Target</th><td>{result['target']}</td></tr>
|
| 141 |
+
<tr><th>Train / test samples</th><td>{result['n_train']:,} / {result['n_test']:,}</td></tr>
|
| 142 |
+
{metrics_rows}
|
| 143 |
+
</table>
|
| 144 |
+
"""
|
| 145 |
+
if plot_b64:
|
| 146 |
+
html += f'<h2>4. Result Plot</h2><img src="data:image/png;base64,{plot_b64}">'
|
| 147 |
+
if imp_b64:
|
| 148 |
+
html += f'<h2>5. Feature Importance</h2><img src="data:image/png;base64,{imp_b64}">'
|
| 149 |
+
html += "</body></html>"
|
| 150 |
+
|
| 151 |
+
with open(HTML_PATH, "w", encoding="utf-8") as f:
|
| 152 |
+
f.write(html)
|
| 153 |
+
return HTML_PATH
|
requirements.txt
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=4.44.0
|
| 2 |
+
pandas>=2.0
|
| 3 |
+
numpy>=1.24
|
| 4 |
+
scikit-learn>=1.3
|
| 5 |
+
matplotlib>=3.7
|
| 6 |
+
openpyxl>=3.1
|
| 7 |
+
pyarrow>=14.0
|
| 8 |
+
fpdf2>=2.7
|