Update app.py

app.py

@@ -1,447 +1,849 @@
-# If needed in Colab uncomment:
-# !pip install --quiet gradio openpyxl pandas
-
+# app.py
+import os
+import io
+import tempfile
+import json
+import math
+import traceback
+from typing import Optional, List, Tuple, Dict, Any
 import pandas as pd
+import numpy as np
 import gradio as gr
-import warnings
-warnings.filterwarnings('ignore')
-
-# ---------------- Load data ----------------
-data_path = "Effort Estimation Sample Data.xlsx"
-sheet1 = pd.read_excel(data_path, sheet_name="Sheet1")
-sheet2 = pd.read_excel(data_path, sheet_name="Sheet2")
-
-# ---------------- Constants ----------------
-phase_display_names = {
-    "Design effort (days)": "Design",
-    "Build effort (days)": "Build",
-    "Testing effort (days)": "Testing",
-    "Post Go Live (days)": "Post Go Live"
+import plotly.express as px
+import plotly.graph_objects as go
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split, GridSearchCV
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from sklearn.pipeline import Pipeline
+from sklearn.impute import SimpleImputer
+from sklearn.feature_selection import RFECV
+from sklearn.linear_model import LinearRegression, Ridge, Lasso, LogisticRegression
+from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
+from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier, GradientBoostingRegressor, GradientBoostingClassifier
+from sklearn.svm import SVR, SVC
+from sklearn.neighbors import KNeighborsClassifier
+from sklearn.naive_bayes import GaussianNB
+from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score, accuracy_score, classification_report, confusion_matrix, roc_auc_score, roc_curve
+from sklearn.exceptions import NotFittedError
+import scipy.stats as stats
+import matplotlib
+matplotlib.use("Agg")  # for headless plotting
+
+# --- lightweight password hashing (stdlib) ---
+import os as _os
+import hashlib, binascii, hmac as _hmac
+
+def generate_password_hash(password: str) -> str:
+    salt = _os.urandom(16)
+    dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+    return binascii.hexlify(salt).decode() + ':' + binascii.hexlify(dk).decode()
+
+def check_password_hash(stored_hash: str, password: str) -> bool:
+    try:
+        salt_hex, dk_hex = stored_hash.split(':')
+    except ValueError:
+        return False
+    salt = binascii.unhexlify(salt_hex)
+    dk = binascii.unhexlify(dk_hex)
+    new_dk = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
+    return _hmac.compare_digest(new_dk, dk)
+# ------------------------------------------------
+
+# -----------------------------
+# Simple in-repo auth store (file-backed)
+# -----------------------------
+AUTH_STORE = "auth_store.json"
+def _ensure_auth():
+    if not os.path.exists(AUTH_STORE):
+        admin_pass = generate_password_hash("password")
+        with open(AUTH_STORE, "w") as f:
+            json.dump({"admin": admin_pass}, f)
+
+def authenticate(username: str, password: str) -> bool:
+    _ensure_auth()
+    with open(AUTH_STORE, "r") as f:
+        data = json.load(f)
+    stored = data.get(username)
+    if not stored:
+        return False
+    return check_password_hash(stored, password)
+
+# seed auth
+_ensure_auth()
+
+# -----------------------------
+# App constants & helpers
+# -----------------------------
+HIGH_CARD_THRESHOLD_COUNT = 50   # or unique > threshold proportion
+HIGH_CARD_THRESHOLD_RATIO = 0.10 # if unique/rows > this, consider high card
+
+LOGO_PATH = "DataSynth.png"  # ensure this file exists in repo root
+
+# -----------------------------
+# Small utility functions
+# -----------------------------
+def read_file_to_df(uploaded) -> pd.DataFrame:
+    """uploaded is a gradio file dict-like or path"""
+    if uploaded is None:
+        return pd.DataFrame()
+    # Uploaded from HF Spaces will be a dict-like with 'name' or 'tmp_path'
+    try:
+        # Gradio on Spaces often gives a tempfile path in `name` or `file`
+        if hasattr(uploaded, "name") and os.path.exists(uploaded.name):
+            path = uploaded.name
+        elif isinstance(uploaded, dict) and "name" in uploaded:
+            path = uploaded["name"]
+        else:
+            # attempt to read bytes
+            content = uploaded.read()
+            try:
+                return pd.read_csv(io.BytesIO(content))
+            except Exception:
+                return pd.read_excel(io.BytesIO(content))
+        # now path-based read
+        if path.lower().endswith(".csv"):
+            return pd.read_csv(path)
+        elif path.lower().endswith((".xls", ".xlsx")):
+            return pd.read_excel(path)
+        elif path.lower().endswith(".json"):
+            return pd.read_json(path)
+        else:
+            # try csv then excel then json
+            try:
+                return pd.read_csv(path)
+            except Exception:
+                try:
+                    return pd.read_excel(path)
+                except Exception:
+                    return pd.read_json(path)
+    except Exception as e:
+        print("read_file_to_df error:", e)
+        raise
+
+def basic_profile(df: pd.DataFrame) -> Dict[str, Any]:
+    if df is None or df.empty:
+        return {}
+    profile = {}
+    profile["rows"], profile["columns"] = df.shape
+    dtypes = df.dtypes.apply(lambda x: x.name).to_dict()
+    profile["dtypes"] = dtypes
+    nulls = df.isnull().sum().to_dict()
+    profile["nulls"] = nulls
+    profile["null_pct"] = {k: (v / len(df)) for k, v in nulls.items()}
+    unique_counts = df.nunique(dropna=False).to_dict()
+    profile["unique"] = unique_counts
+    profile["high_cardinality"] = [col for col, cnt in unique_counts.items()
+                                  if cnt > HIGH_CARD_THRESHOLD_COUNT or (cnt / len(df) > HIGH_CARD_THRESHOLD_RATIO)]
+    profile["describe"] = df.describe(include='all').to_dict()
+    profile["head"] = df.head(5).to_dict(orient="records")
+    return profile
+
+def profile_to_markdown(profile: Dict[str, Any]) -> str:
+    if not profile:
+        return "No data loaded."
+    md = []
+    md.append(f"**Rows:** {profile['rows']}  \n**Columns:** {profile['columns']}\n")
+    md.append("### Column summary (dtypes / null% / unique)\n")
+    md.append("| Column | Dtype | Nulls | Null % | Unique |\n|---:|---|---:|---:|---:|\n")
+    for col in profile["dtypes"].keys():
+        dtype = profile["dtypes"][col]
+        nulls = profile["nulls"].get(col, 0)
+        pct = f"{profile['null_pct'].get(col,0):.2%}"
+        uniq = profile["unique"].get(col, 0)
+        md.append(f"| {col} | {dtype} | {nulls} | {pct} | {uniq} |\n")
+    if profile["high_cardinality"]:
+        md.append("\n**High cardinality columns (auto-detected):** " + ", ".join(profile["high_cardinality"]) + "\n")
+    md.append("\n### Sample rows\n")
+    md.append(pd.DataFrame(profile["head"]).to_markdown(index=False))
+    return "\n".join(md)
+
+# -----------------------------
+# Data cleaning & feature engineering helpers
+# -----------------------------
+def drop_high_cardinality(df: pd.DataFrame, threshold_count=HIGH_CARD_THRESHOLD_COUNT, threshold_ratio=HIGH_CARD_THRESHOLD_RATIO):
+    n = len(df)
+    cols_to_drop = []
+    for c in df.columns:
+        if df[c].nunique(dropna=False) > threshold_count or (df[c].nunique(dropna=False)/max(1,n) > threshold_ratio):
+            # drop object/categorical high-card only (keep numeric)
+            if df[c].dtype == "object" or str(df[c].dtype).startswith("category"):
+                cols_to_drop.append(c)
+    return df.drop(columns=cols_to_drop, errors='ignore'), cols_to_drop
+
+def impute_df(df: pd.DataFrame, numeric_strategy="mean", categorical_strategy="most_frequent", fill_value: Optional[str]=None):
+    df = df.copy()
+    num_cols = df.select_dtypes(include=[np.number]).columns.tolist()
+    cat_cols = df.select_dtypes(include=["object", "category"]).columns.tolist()
+    if num_cols:
+        imp = SimpleImputer(strategy=numeric_strategy)
+        df[num_cols] = imp.fit_transform(df[num_cols])
+    if cat_cols:
+        if categorical_strategy == "constant" and fill_value is not None:
+            imp2 = SimpleImputer(strategy="constant", fill_value=fill_value)
+        else:
+            imp2 = SimpleImputer(strategy=categorical_strategy)
+        df[cat_cols] = imp2.fit_transform(df[cat_cols])
+    return df
+
+def treat_outliers_iqr(df: pd.DataFrame, cols: List[str], method="cap"):
+    df = df.copy()
+    for c in cols:
+        if c not in df.columns:
+            continue
+        if not np.issubdtype(df[c].dtype, np.number):
+            continue
+        q1 = df[c].quantile(0.25)
+        q3 = df[c].quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - 1.5 * iqr
+        upper = q3 + 1.5 * iqr
+        if method == "remove":
+            df = df[(df[c] >= lower) & (df[c] <= upper)]
+        elif method == "cap":
+            df[c] = np.where(df[c] < lower, lower, df[c])
+            df[c] = np.where(df[c] > upper, upper, df[c])
+    return df
+
+def parse_dates(df: pd.DataFrame, col: str, fmt: Optional[str]=None):
+    df = df.copy()
+    try:
+        if fmt:
+            df[col] = pd.to_datetime(df[col], format=fmt, errors="coerce")
+        else:
+            df[col] = pd.to_datetime(df[col], errors="coerce", infer_datetime_format=True)
+    except Exception as e:
+        print("parse_dates", e)
+    return df
+
+def text_clean(df: pd.DataFrame, cols: List[str], lower=True, strip=True):
+    df = df.copy()
+    for c in cols:
+        if c not in df.columns:
+            continue
+        df[c] = df[c].astype(str)
+        if strip:
+            df[c] = df[c].str.strip()
+        if lower:
+            df[c] = df[c].str.lower()
+    return df
+
+def transform_cols(df: pd.DataFrame, cols: List[str], method="log"):
+    df = df.copy()
+    for c in cols:
+        if c in df.columns and np.issubdtype(df[c].dtype, np.number):
+            if method == "log":
+                df[c] = df[c].apply(lambda x: np.log(x) if x>0 else x)
+            elif method == "sqrt":
+                df[c] = df[c].apply(lambda x: np.sqrt(x) if x>=0 else x)
+    return df
+
+# -----------------------------
+# Visualization NLP (very small parser)
+# -----------------------------
+def nlp_to_chart_instruction(query: str, df: pd.DataFrame):
+    """Return (chart_type, cols) based on keywords"""
+    q = query.lower()
+    numeric = df.select_dtypes(include=[np.number]).columns.tolist()
+    categorical = df.select_dtypes(include=["object", "category"]).columns.tolist()
+    tokens = q.split()
+    # histogram
+    if "hist" in q or "histogram" in q or "distribution" in q:
+        # pick first numeric mention
+        for c in numeric:
+            if c.lower() in q:
+                return ("hist", [c])
+        if numeric:
+            return ("hist", [numeric[0]])
+    # scatter
+    if "scatter" in q or "vs" in q or "versus" in q:
+        # pick first two numeric
+        for c in numeric:
+            if c.lower() in q:
+                x = c
+                # find second numeric
+                for d in numeric:
+                    if d!=c and d.lower() in q:
+                        return ("scatter", [x, d])
+                # fallback second numeric
+                if len(numeric)>1:
+                    return ("scatter", [numeric[0], numeric[1]])
+        if len(numeric)>=2:
+            return ("scatter", [numeric[0], numeric[1]])
+    # bar chart for categorical counts
+    if "bar" in q or "count" in q or "counts" in q or "value counts" in q:
+        for c in categorical:
+            if c.lower() in q:
+                return ("bar", [c])
+        if categorical:
+            return ("bar", [categorical[0]])
+    # boxplot
+    if "box" in q or "outlier" in q:
+        for c in numeric:
+            if c.lower() in q:
+                return ("box", [c])
+        if numeric:
+            return ("box", [numeric[0]])
+    # fallback: table head
+    return ("table", [])
+
+def render_chart_from_instruction(instruction: Tuple[str, List[str]], df: pd.DataFrame):
+    typ, cols = instruction
+    if typ == "hist":
+        c = cols[0]
+        fig = px.histogram(df, x=c, title=f"Distribution of {c}")
+        return fig
+    if typ == "scatter":
+        x, y = cols[:2]
+        fig = px.scatter(df, x=x, y=y, title=f"{y} vs {x}")
+        return fig
+    if typ == "bar":
+        c = cols[0]
+        vc = df[c].value_counts().reset_index()
+        vc.columns = [c, "count"]
+        fig = px.bar(vc, x=c, y="count", title=f"Counts of {c}")
+        return fig
+    if typ == "box":
+        c = cols[0]
+        fig = px.box(df, y=c, title=f"Box plot of {c}")
+        return fig
+    # table
+    return None
+
+# -----------------------------
+# Modeling helpers
+# -----------------------------
+REGRESSION_MODELS = {
+    "LinearRegression": LinearRegression,
+    "Ridge": Ridge,
+    "Lasso": Lasso,
+    "DecisionTreeRegressor": DecisionTreeRegressor,
+    "RandomForestRegressor": RandomForestRegressor,
+    "GradientBoostingRegressor": GradientBoostingRegressor,
+    "SVR": SVR
 }
 
-EY_COLORS = {
-    "black": "#333333",
-    "white": "#FFFFFF",
-    "turbo": "#FFE600",
-    "sonic_silver": "#797878",
-    "sizzling_sunrise": "#FFDB00"
+CLASSIFICATION_MODELS = {
+    "LogisticRegression": LogisticRegression,
+    "DecisionTreeClassifier": DecisionTreeClassifier,
+    "RandomForestClassifier": RandomForestClassifier,
+    "GradientBoostingClassifier": GradientBoostingClassifier,
+    "SVC": SVC,
+    "GaussianNB": GaussianNB,
+    "KNN": KNeighborsClassifier
 }
 
-# Prepare initial lists
-competencies = sorted(list(sheet1["Competency"].dropna().unique()))
-
-# ---------------- Helper functions (unchanged core logic) ----------------
-def get_modules(competency):
-    if not competency:
-        return []
-    return list(sheet1[sheet1["Competency"] == competency]["Module"].dropna().unique())
-
-def get_submodules(competency, modules):
-    if not competency or not modules:
-        return []
-    all_submodules = []
-    for module in modules:
-        submodules = list(sheet1[
-            (sheet1["Competency"] == competency) &
-            (sheet1["Module"] == module)
-        ]["Sub - Module"].dropna().unique())
-        all_submodules.extend(submodules)
-    return all_submodules
-
-def get_complexities_for_submodule(competency, module, submodule):
-    if not competency or not module or not submodule:
-        return []
-    return list(sheet1[
-        (sheet1["Competency"] == competency) & (sheet1["Module"] == module) & (sheet1["Sub - Module"] == submodule)
-    ]["Complexity Level"].dropna().unique())
-
-def get_phases_for_submodule(competency, module, submodule):
-    if not competency or not module or not submodule:
-        return []
-    phases = []
-    filtered = sheet1[
-        (sheet1["Competency"] == competency) & (sheet1["Module"] == module) & (sheet1["Sub - Module"] == submodule)
-    ]
-    if not filtered.empty:
-        for phase_col, disp in phase_display_names.items():
-            if phase_col in filtered.columns and filtered[phase_col].sum() > 0:
-                phases.append(disp)
-    return phases
-
-def create_submodule_details_html(competency, modules, submodules):
-    """Read-only available complexities & phases per submodule (visual aid)."""
-    if not competency or not modules or not submodules:
-        return ""
-    html = f"""
-    <div style='background: {EY_COLORS["white"]}; padding: 12px; border-radius: 8px; margin: 6px 0; border: 1px solid {EY_COLORS["sonic_silver"]};'>
-        <h4 style='color: {EY_COLORS["black"]}; margin-top: 0;'>📋 Sub-Module Details (Available)</h4>
-    """
-    for module in modules:
-        for submodule in submodules:
-            complexities = get_complexities_for_submodule(competency, module, submodule)
-            phases = get_phases_for_submodule(competency, module, submodule)
-            if complexities or phases:
-                html += f"""
-                <div style='padding:10px;margin:6px 0;border-radius:6px;border-left:4px solid {EY_COLORS['turbo']};background:{EY_COLORS['white']};'>
-                    <strong style='color:{EY_COLORS['black']};'>{module} → {submodule}</strong><br>
-                    <span style='color:{EY_COLORS['sonic_silver']}; font-size:0.95em;'>
-                        🎯 Complexities: {', '.join(complexities) if complexities else 'None'}<br>
-                        📋 Phases: {', '.join(phases) if phases else 'None'}
-                    </span>
-                </div>
-                """
-    html += "</div>"
-    return html
-
-# ---------------- Calculation using per-submodule selections (keeps original formula & HTML) ----------------
-def calculate_effort_per_submodule(competency, selected_modules, selected_submodules, submodule_counts, per_submodule_complexities, per_submodule_phases, buffer_percent):
-    """
-    per_submodule_complexities and per_submodule_phases are dicts keyed by submodule name -> list
-    """
-    if not competency:
-        return "❌ Please select a Competency."
-    if not selected_modules:
-        return "❌ Please select at least one Module."
-    if not selected_submodules:
-        return "❌ Please select at least one Sub-Module."
-
-    # Validate at least one complexity & phase chosen across selected submodules
-    overall_complexities = set()
-    overall_phases = set()
-    for sm in selected_submodules:
-        cs = per_submodule_complexities.get(sm, [])
-        ps = per_submodule_phases.get(sm, [])
-        overall_complexities.update(cs)
-        overall_phases.update(ps)
-    if not overall_complexities:
-        return "❌ Please select at least one Complexity Level for the chosen sub-modules."
-    if not overall_phases:
-        return "❌ Please select at least one Phase for the chosen sub-modules."
-
-    total_effort = 0.0
-    breakdown_rows = []
-
-    for module in selected_modules:
-        for submodule in selected_submodules:
-            count = int(submodule_counts.get(submodule, 1))
-            complexities = per_submodule_complexities.get(submodule, [])
-            phases = per_submodule_phases.get(submodule, [])
-            for complexity in complexities:
-                filtered_data = sheet1[
-                    (sheet1["Competency"] == competency) &
-                    (sheet1["Module"] == module) &
-                    (sheet1["Sub - Module"] == submodule) &
-                    (sheet1["Complexity Level"] == complexity)
-                ]
-                if filtered_data.empty:
-                    continue
-
-                weight_rows = sheet2[sheet2["Complexity"] == complexity]
-                weight = float(weight_rows["Weightage"].values[0]) if not weight_rows.empty else 1.0
-
-                combination_effort = 0.0
-                phase_details = []
-                for phase in phases:
-                    phase_col = next((col for col, disp in phase_display_names.items() if disp == phase), None)
-                    if phase_col and phase_col in filtered_data.columns:
-                        phase_effort = float(filtered_data[phase_col].sum()) * weight
-                        phase_total = phase_effort * count
-                        combination_effort += phase_total
-                        phase_details.append(f"{phase}: {phase_effort:.1f}x{count} = {phase_total:.1f} days")
-
-                if combination_effort > 0:
-                    total_effort += combination_effort
-                    breakdown_rows.append({
-                        'module': module,
-                        'submodule': submodule,
-                        'complexity': complexity,
-                        'count': count,
-                        'phases': ', '.join(phases),
-                        'effort': round(combination_effort, 2),
-                        'phase_details': ' | '.join(phase_details)
-                    })
-
-    if total_effort == 0:
-        return "❌ No effort data found for the selected combinations."
-
-    final_effort = total_effort * (1 + buffer_percent / 100)
-
-    # Result HTML exactly following your original template
-    result = f"""
-<div style='font-family: Arial, sans-serif; color: {EY_COLORS["black"]};'>
-    <h1 style='color: {EY_COLORS["black"]}; border-bottom: 2px solid {EY_COLORS["sizzling_sunrise"]}; padding-bottom: 10px;'>
-        📊 EY Effort Estimation Results
-    </h1>
-    <div style='background: {EY_COLORS["turbo"]}20; padding: 15px; border-radius: 5px; margin: 10px 0;'>
-        <strong>Competency:</strong> {competency}<br>
-        <strong>Selected Modules:</strong> {', '.join(selected_modules)}<br>
-        <strong>Buffer Percentage:</strong> {buffer_percent}%
-    </div>
-    <h2 style='color: {EY_COLORS["black"]};'>Effort Breakdown</h2>
-    <table style='width: 100%; border-collapse: collapse; border: 2px solid {EY_COLORS["sonic_silver"]};'>
-        <tr style='background: {EY_COLORS["sizzling_sunrise"]};'>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: left;'>Module</th>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: left;'>Sub Module</th>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: left;'>Complexity</th>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: left;'>Count</th>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: left;'>Phases</th>
-            <th style='padding: 12px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: right;'>Effort (days)</th>
-        </tr>
-"""
-    for i, row in enumerate(breakdown_rows):
-        bg_color = f"{EY_COLORS['white']}" if i % 2 == 0 else f"{EY_COLORS['turbo']}20"
-        result += f"""
-        <tr style='background: {bg_color};'>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]};'>{row['module']}</td>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]};'>{row['submodule']}</td>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]};'>{row['complexity']}</td>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]};'>{row['count']}</td>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]};'>{row['phases']}</td>
-            <td style='padding: 10px; border: 1px solid {EY_COLORS["sonic_silver"]}; text-align: right;'>{row['effort']}</td>
-        </tr>
+def prepare_features_targets(df: pd.DataFrame, target: str, drop_cols: List[str]=None, drop_high_card=True):
+    df = df.copy()
+    if drop_cols:
+        df = df.drop(columns=drop_cols, errors='ignore')
+    if drop_high_card:
+        df, dropped = drop_high_cardinality(df)
+    if target not in df.columns:
+        raise ValueError("Target column not in dataframe")
+    X = df.drop(columns=[target])
+    y = df[target]
+    return X, y
+
+def auto_build_preprocessor(X: pd.DataFrame, scaler_choice: str="standard", onehot=True):
+    num_cols = X.select_dtypes(include=[np.number]).columns.tolist()
+    cat_cols = X.select_dtypes(include=["object", "category"]).columns.tolist()
+    transformers = []
+    if num_cols:
+        if scaler_choice == "standard":
+            num_pipeline = Pipeline([("imputer", SimpleImputer(strategy="mean")), ("scaler", StandardScaler())])
+        elif scaler_choice == "minmax":
+            num_pipeline = Pipeline([("imputer", SimpleImputer(strategy="mean")), ("scaler", MinMaxScaler())])
+        else:
+            num_pipeline = Pipeline([("imputer", SimpleImputer(strategy="mean"))])
+        transformers.append(("num", num_pipeline, num_cols))
+    if cat_cols and onehot:
+        cat_pipeline = Pipeline([("imputer", SimpleImputer(strategy="most_frequent")), ("onehot", OneHotEncoder(drop='first', sparse=False, handle_unknown='ignore'))])
+        transformers.append(("cat", cat_pipeline, cat_cols))
+    preprocessor = ColumnTransformer(transformers=transformers, remainder='drop')
+    return preprocessor
+
+def fit_and_evaluate_model(X, y, model_name: str, task:str, scaler_choice="standard", test_size=0.2, random_state=42, do_rfecv=False, param_grid=None):
+    if task=="regression":
+        ModelClass = REGRESSION_MODELS.get(model_name)
+    else:
+        ModelClass = CLASSIFICATION_MODELS.get(model_name)
+
+    if ModelClass is None:
+        raise ValueError(f"Model {model_name} not found for task {task}")
+
+    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=test_size, random_state=random_state)
+    preprocessor = auto_build_preprocessor(X_train, scaler_choice=scaler_choice, onehot=True)
+    model = ModelClass()
+    pipe = Pipeline([("pre", preprocessor), ("model", model)])
+
+    # optional RFECV on a simple estimator if requested (only if classifier/regressor supports coef_ or feature_importances_)
+    rfecv_result = None
+    if do_rfecv:
+        try:
+            # use a wrapped estimator for feature selection (random forest)
+            base_est = RandomForestRegressor(n_estimators=20) if task=="regression" else RandomForestClassifier(n_estimators=20)
+            rfecv = RFECV(estimator=base_est, step=1, cv=3, scoring='r2' if task=="regression" else 'accuracy', n_jobs=1)
+            # build simple pipeline: preprocessor -> rfecv will need transformed X
+            X_trans = preprocessor.fit_transform(X_train)
+            rfecv.fit(X_trans, y_train)
+            rfecv_result = rfecv
+            # use selected features by indices (skipping because transformer may produce different width)
+        except Exception as e:
+            print("RFECV failed:", e)
+
+    # grid search if provided
+    gs = None
+    if param_grid:
+        gs = GridSearchCV(pipe, param_grid, cv=3, n_jobs=1)
+        gs.fit(X_train, y_train)
+        best_est = gs.best_estimator_
+        y_pred = best_est.predict(X_test)
+        trained = best_est
+    else:
+        pipe.fit(X_train, y_train)
+        y_pred = pipe.predict(X_test)
+        trained = pipe
+
+    results = {}
+    # metrics
+    if task=="regression":
+        results["mse"] = mean_squared_error(y_test, y_pred)
+        results["rmse"] = math.sqrt(results["mse"])
+        results["mae"] = mean_absolute_error(y_test, y_pred)
+        results["r2"] = r2_score(y_test, y_pred)
+        # residuals and QQ
+        resid = y_test - y_pred
+        results["residuals"] = resid
+    else:
+        results["accuracy"] = accuracy_score(y_test, y_pred)
+        results["report"] = classification_report(y_test, y_pred, output_dict=True)
+        # ROC if probability available
+        try:
+            if hasattr(trained.named_steps['model'], "predict_proba"):
+                y_score = trained.predict_proba(X_test)[:,1]
+                results["roc_auc"] = roc_auc_score(y_test, y_score)
+                fpr, tpr, _ = roc_curve(y_test, y_score)
+                results["roc_curve"] = (fpr.tolist(), tpr.tolist())
+        except Exception as e:
+            print("ROC error", e)
+        results["confusion_matrix"] = confusion_matrix(y_test, y_pred).tolist()
+
+    # feature importance if available (for tree models)
+    feat_importance = None
+    try:
+        model_obj = trained.named_steps['model']
+        if hasattr(model_obj, "feature_importances_"):
+            # get feature names after preprocessor
+            # try to extract feature names from preprocessor
+            feature_names = []
+            pre = trained.named_steps['pre']
+            if hasattr(pre, 'transformers_'):
+                for name, trans, cols in pre.transformers_:
+                    if name == "num":
+                        feature_names += cols
+                    elif name == "cat":
+                        # extract ohe names
+                        ohe = trans.named_steps['onehot']
+                        if hasattr(ohe, 'get_feature_names_out'):
+                            names = list(ohe.get_feature_names_out(cols))
+                            feature_names += names
+                        else:
+                            feature_names += cols
+            importances = model_obj.feature_importances_
+            feat_importance = list(zip(feature_names, importances))
+            feat_importance.sort(key=lambda x: x[1], reverse=True)
+    except Exception as e:
+        print("feature importance error", e)
+
+    results["feature_importance"] = feat_importance
+    results["trained"] = trained
+    results["rfecv"] = rfecv_result
+    results["y_test_sample"] = None
+    return results
+
+# -----------------------------
+# Gradio UI
+# -----------------------------
+css = """
+/* Vanta background container */
+#vanta-bg {
+  width: 100%;
+  height: 380px;
+  position: relative;
+  overflow: hidden;
+  border-radius: 12px;
+  margin-bottom: 8px;
+}
+
+/* overlay login card */
+.login-card {
+  position: absolute;
+  left: 50%;
+  top: 50%;
+  transform: translate(-50%, -50%);
+  width: 420px;
+  max-width: calc(100% - 24px);
+  background: linear-gradient(180deg, rgba(255,255,255,0.98), rgba(245,245,255,0.95));
+  border-radius: 14px;
+  box-shadow: 0 12px 36px rgba(0,0,0,0.18);
+  padding: 22px;
+  z-index: 999;
+  border: 1px solid rgba(0,0,0,0.06);
+}
+
+/* logo */
+.login-logo {
+  display:flex;
+  align-items:center;
+  gap:12px;
+  margin-bottom:8px;
+}
+
+.brand-title {
+  font-weight:700;
+  font-size:18px;
+  color:#2b2b6b;
+}
+
+/* button animation */
+.btn-animate {
+  transition: transform 0.12s ease-in-out, box-shadow 0.12s;
+}
+.btn-animate:active {
+  transform: translateY(2px) scale(0.995);
+  box-shadow: 0 6px 18px rgba(0,0,0,0.12) inset;
+}
+
+/* small description text */
+.app-desc {
+  font-size: 13px;
+  color: #444;
+  margin-top: 8px;
+  text-align: center;
+}
 """
-    result += f"""
-    </table>
-    <div style='margin-top: 20px; padding: 15px; background: {EY_COLORS["white"]}; border: 2px solid {EY_COLORS["sonic_silver"]}; border-radius: 5px;'>
-        <table style='width: 100%;'>
-            <tr>
-                <td style='padding: 10px;'><strong>Total Efforts (before buffer):</strong></td>
-                <td style='padding: 10px; text-align: right;'>{total_effort:.2f} days</td>
-            </tr>
-            <tr>
-                <td style='padding: 10px;'><strong>Buffer Applied ({buffer_percent}%):</strong></td>
-                <td style='padding: 10px; text-align: right;'>+ {total_effort * buffer_percent/100:.2f} days</td>
-            </tr>
-            <tr style='background: {EY_COLORS["sizzling_sunrise"]};'>
-                <td style='padding: 15px;'><strong>Total Efforts (with buffer {buffer_percent}%):</strong></td>
-                <td style='padding: 15px; text-align: right; font-size: 1.2em;'><strong>{final_effort:.2f} days</strong></td>
-            </tr>
-        </table>
+
+vanta_html = f"""
+<div id="vanta-bg" style="width:100%;height:380px;border-radius:12px;position:relative;">
+  <div class="login-card" role="region" aria-label="Login card">
+    <div class="login-logo">
+      <img src="{LOGO_PATH}" alt="logo" style="height:48px;width:48px;border-radius:8px;"/>
+      <div>
+        <div class="brand-title">DataSynth — Analytics Hub</div>
+        <div style="font-size:12px;color:#666;">Fast, modular data profiling & model building</div>
+      </div>
     </div>
-    <div style='margin-top: 20px; padding: 15px; background: {EY_COLORS["turbo"]}20; border-radius: 5px;'>
-        <h3>Detailed Phase Breakdown:</h3>
-"""
-    for row in breakdown_rows:
-        result += f"""
-        <p><strong>{row['module']} - {row['submodule']} ({row['complexity']}):</strong> {row['phase_details']}</p>
-"""
-    result += """
+    <div style="margin-top:8px;">
+      <div style="font-size:13px;color:#333;margin-bottom:6px;">Sign in to continue</div>
     </div>
+    <!-- gradio inputs are rendered below visually; this card serves as overlay -->
+    <div class="app-desc">Upload your dataset, prepare it, visualize with NL, and build ML models — all in one place.</div>
+  </div>
 </div>
+
+<!-- Vanta & three.js from CDN -->
+<script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r121/three.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/vanta@0.5.21/dist/vanta.net.min.js"></script>
+<script>
+  (function(){
+    try {
+      if (typeof VANTA !== 'undefined') {
+        VANTA.NET({
+          el: "#vanta-bg",
+          color: 0x2b2b6b,
+          backgroundColor: 0xffffff,
+          points: 10.00,
+          maxDistance: 26.00
+        })
+      }
+    } catch(e) {
+      console.warn("Vanta failed to initialize", e)
+    }
+  })();
+</script>
 """
-    return result
-
-# ---------------- UI: pre-create rows for submodules (scalable) ----------------
-MAX_SUBMODULES = 30   # increase if needed
-
-with gr.Blocks(
-    theme=gr.themes.Default(primary_hue="yellow", neutral_hue="gray"),
-    css=f"""
-    .gradio-container {{
-        background: linear-gradient(135deg, {EY_COLORS['white']} 0%, {EY_COLORS['turbo']}10 100%);
-        font-family: Arial, sans-serif;
-    }}
-    .ey-header {{
-        text-align: center; padding: 18px; color: {EY_COLORS['black']};
-    }}
-    .ey-header img {{ height: 54px; margin-right: 12px; vertical-align: middle; }}
-    .ey-button {{ background: {EY_COLORS['turbo']} !important; color: {EY_COLORS['black']} !important; border: 1px solid {EY_COLORS['sonic_silver']} !important; }}
-    .panel {{ padding: 16px; background: {EY_COLORS['white']}; border-radius: 10px; margin: 8px; box-shadow: 0 4px 12px rgba(0,0,0,0.06); }}
-    .sub-row {{ padding: 6px 0; }}
-    """
-) as demo:
-
-    # Header
-    gr.HTML(f"""
-    <div class="ey-header">
-        <img src="EY Logo.png" alt="EY Logo"
-             onerror="this.src='https://www.ey.com/content/dam/ey-unified-site/ey-com/en-in/generic/images/ey-logo-black.png'">
-        <div style="display:inline-block; vertical-align:middle;">
-            <div style="font-size:26px; font-weight:700; color:{EY_COLORS['black']};">Effort Estimation Tool</div>
-            <div style="color:{EY_COLORS['sonic_silver']}; margin-top:3px;">Enterprise Grade Estimation Platform</div>
-        </div>
-    </div>
-    """)
 
-    # ----------------- Page 1: Inputs (left / right) -----------------
-    with gr.Column(visible=True) as page_inputs:
+# Gradio app layout
+with gr.Blocks(css=css, title="DataSynth — Analytics Hub") as demo:
+    # --- Login area ---
+    with gr.Column():
+        v_html = gr.HTML(vanta_html)
+        with gr.Row():
+            username_in = gr.Textbox(label="Username", placeholder="username", interactive=True)
+            password_in = gr.Textbox(label="Password", placeholder="password", type="password")
         with gr.Row():
-            # Left: selectors & sub-modules
-            with gr.Column(scale=1, elem_classes="panel"):
-                # Competency (no default)
-                competency = gr.Dropdown(choices=competencies, value=None, label="🏢 Select Competency", interactive=True)
-
-                # Modules will populate after competency chosen
-                modules = gr.CheckboxGroup(choices=[], value=[], label="📦 Select Modules", interactive=True)
-
-                # Submodule rows area (pre-created hidden rows)
-                gr.Markdown("### 🔧 Sub-module level configuration")
-                gr.Markdown("<small>🔔 Select only one Complexity level per sub-module. Phases can be multi-selected.</small>")
-
-                per_sm_row = []
-                per_sm_checkbox = []
-                per_sm_name = []
-                per_sm_count = []
-                per_sm_complexities = []
-                per_sm_phases = []
-
-                for i in range(MAX_SUBMODULES):
-                    with gr.Row(visible=False) as row:
-                        chk = gr.Checkbox(label="", value=False)                          # select to include
-                        name = gr.Textbox(value="", interactive=False, label="", visible=True)  # submodule name display
-                        cnt = gr.Number(value=1, minimum=1, label="Count", interactive=True)   # inline count
-                        # per-submodule complexity (single choice) & phases controls
-                        comps_radio = gr.Radio(choices=[], value=None, label="Complexity (select one)", interactive=True)
-                        phases_cg = gr.CheckboxGroup(choices=[], value=[], label="Phases", interactive=True)
-                    per_sm_row.append(row)
-                    per_sm_checkbox.append(chk)
-                    per_sm_name.append(name)
-                    per_sm_count.append(cnt)
-                    per_sm_complexities.append(comps_radio)
-                    per_sm_phases.append(phases_cg)
-
-                # Submodule details HTML (available options summary)
-                submodule_details = gr.HTML("")
-
-            # Right: Buffer, Calculate & (no results here) 
-            with gr.Column(scale=1, elem_classes="panel"):
-                buffer_percent = gr.Slider(0, 50, value=16, step=1, label="📊 Buffer Percentage (%)")
-                calculate_btn = gr.Button("🚀 Calculate Total Effort", elem_classes="ey-button")
-
-    # ----------------- Page 2: Results (hidden initially) -----------------
-    with gr.Column(visible=False, elem_classes="panel") as page_results:
-        # Results area with vertical scroll style
-        results = gr.HTML("", label="Estimation Results")
-        back_btn = gr.Button("⬅️ Back to Inputs")
-
-    # ------------------- Event handlers -------------------
-
-    # When competency changes -> populate modules (no default competency)
-    def on_competency_change(comp):
-        return gr.update(choices=get_modules(comp), value=[])
-
-    competency.change(fn=on_competency_change, inputs=competency, outputs=modules)
-
-    # When modules change -> populate submodule rows, update per-row complexity/phase choices & details html
-    def on_modules_change(comp, mods):
-        submods = get_submodules(comp, mods) if mods else []
-        updates = []
-
-        for i in range(MAX_SUBMODULES):
-            if i < len(submods):
-                sm_name = submods[i]
-                # show row
-                updates.append(gr.update(visible=True))                         # row
-                updates.append(gr.update(label=sm_name, value=False))           # checkbox label & unchecked
-                updates.append(gr.update(value=sm_name))                        # name textbox value
-                updates.append(gr.update(visible=True, value=1))                # count visible default 1
-
-                # per-submodule complexities & phases
-                # find module for this submodule (choose from selected modules that contains this submodule)
-                module_for_sm = None
-                for m in mods:
-                    if sm_name in list(sheet1[(sheet1["Competency"] == comp) & (sheet1["Module"] == m)]["Sub - Module"].dropna().unique()):
-                        module_for_sm = m
-                        break
-                comp_choices = get_complexities_for_submodule(comp, module_for_sm, sm_name) if module_for_sm else []
-                phase_choices = get_phases_for_submodule(comp, module_for_sm, sm_name) if module_for_sm else []
-
-                # Complexity is now Radio -> don't pre-select (value=None)
-                updates.append(gr.update(choices=comp_choices, value=None))
-                updates.append(gr.update(choices=phase_choices, value=phase_choices))
+            login_btn = gr.Button("Log in", elem_classes="btn-animate")
+            login_msg = gr.Label(value="")
+        # state holders for data and cleaned data and metadata
+        raw_state = gr.State(value=None)      # will hold JSON-serializable representation (like csv bytes saved to temp file path)
+        df_state = gr.State(value=None)       # original dataframe serialized as json (for safety we store as csv bytes path below)
+        clean_state = gr.State(value=None)    # cleaned dataframe path
+        profile_state = gr.State(value=None)
+
+    # --- Main app (hidden until login) ---
+    with gr.Column(visible=False) as main_area:
+        gr.Markdown("## Workspace")
+        with gr.Tabs():
+            with gr.TabItem("Data"):
+                gr.Markdown("Upload CSV / Excel / JSON for profiling and cleaning.")
+                upload = gr.File(label="Upload CSV / Excel / JSON", file_types=[".csv", ".xlsx", ".xls", ".json"])
+                with gr.Row():
+                    load_btn = gr.Button("Load & Profile")
+                    download_raw_btn = gr.Button("Download Raw CSV")
+                profile_md = gr.Markdown("No dataset loaded.")
+                sample_table = gr.Dataframe(interactive=False)
+            with gr.TabItem("Prepare"):
+                gr.Markdown("Data cleaning and feature engineering options.")
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        impute_num = gr.Dropdown(choices=["mean","median","most_frequent"], value="mean", label="Numeric Imputation")
+                        impute_cat = gr.Dropdown(choices=["most_frequent","constant"], value="most_frequent", label="Categorical Imputation")
+                        const_fill = gr.Textbox(label="Constant fill value (if constant chosen)", value="missing")
+                        outlier_cols = gr.Textbox(label="Outlier numeric columns (comma separated) — leave blank for auto numeric")
+                        outlier_method = gr.Dropdown(choices=["cap","remove"], value="cap", label="Outlier treatment")
+                        date_col = gr.Dropdown(choices=[], label="Date column (auto-detected)", interactive=True)
+                        date_fmt = gr.Textbox(label="Date format (optional)", placeholder="%Y-%m-%d")
+                        text_cols = gr.Textbox(label="Text columns to clean (comma separated)")
+                        transform_cols_txt = gr.Textbox(label="Numeric columns to transform (comma separated)")
+                        transform_method = gr.Dropdown(choices=["log","sqrt"], value="log", label="Transform method")
+                        drop_highcard = gr.Checkbox(label="Auto-drop high-cardinality categorical columns", value=True)
+                        apply_prep = gr.Button("Apply Preparation")
+                        download_clean = gr.Button("Download Clean CSV")
+                    with gr.Column(scale=1):
+                        prep_output = gr.Markdown("Preparation preview will appear here.")
+                        preview_clean = gr.Dataframe(interactive=False)
+            with gr.TabItem("Visualize (NL)"):
+                gr.Markdown("Type a natural-language request to create a plot (e.g., 'histogram of age', 'scatter income vs age', 'bar of country').")
+                nl_input = gr.Textbox(label="Describe chart")
+                nl_btn = gr.Button("Create Chart")
+                nl_plot = gr.Plot()
+            with gr.TabItem("Model"):
+                gr.Markdown("Choose task, target variable, and model. Scaling and encoding will be applied automatically.")
+                task_select = gr.Radio(choices=["regression","classification"], value="regression", label="Task")
+                target_col = gr.Dropdown(choices=[], label="Target variable")
+                scaler_choice = gr.Dropdown(choices=["standard","minmax","none"], value="standard", label="Scaler for numeric")
+                model_select = gr.Dropdown(choices=[*REGRESSION_MODELS.keys(), *CLASSIFICATION_MODELS.keys()], value="RandomForestRegressor", label="Model")
+                rfecv_opt = gr.Checkbox(label="Run RFECV (feature selection)", value=False)
+                train_btn = gr.Button("Train & Evaluate")
+                model_out_md = gr.Markdown("Model results will show here.")
+                model_feature_imp = gr.Dataframe(interactive=False)
+                model_plots = gr.Plot()
+            with gr.TabItem("Report"):
+                gr.Markdown("Generate a short executive report that summarizes profiling, cleaning, and model results.")
+                report_btn = gr.Button("Generate Report")
+                report_download = gr.File(label="Download Report (.md)")
+
+    # --- Callbacks ---
+    def _do_login(username, password):
+        if not username or not password:
+            return gr.update(value="Enter username and password"), gr.update(visible=False)
+        ok = authenticate(username.strip(), password.strip())
+        if ok:
+            return gr.update(value=f"Welcome — {username}"), gr.update(visible=True)
+        else:
+            return gr.update(value="Invalid credentials"), gr.update(visible=False)
+
+    login_btn.click(fn=_do_login, inputs=[username_in, password_in], outputs=[login_msg, main_area])
+
+    # Load & profile dataset
+    def _load_and_profile(uploaded):
+        try:
+            if uploaded is None:
+                return gr.update(value="No file uploaded."), pd.DataFrame(), None, None
+            df = read_file_to_df(uploaded)
+            prof = basic_profile(df)
+            md = profile_to_markdown(prof)
+            # prepare choices for date and target selectors
+            cols = df.columns.tolist()
+            # save df to temp csv for persistence (store path in state)
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv")
+            df.to_csv(tmp.name, index=False)
+            return md, df.head(100), tmp.name, prof
+        except Exception as e:
+            return gr.update(value=f"Error loading file: {e}"), pd.DataFrame(), None, None
+
+    load_btn.click(fn=_load_and_profile, inputs=[upload], outputs=[profile_md, sample_table, df_state, profile_state])
+
+    # download raw
+    def _download_raw(df_path):
+        if not df_path:
+            return None
+        return df_path
+    download_raw_btn.click(fn=_download_raw, inputs=[df_state], outputs=[download_raw_btn])
+
+    # prepare: auto populate date and target choices when profile updated
+    def _populate_prepare(profile):
+        if not profile:
+            return [], []
+        # parse back to usable structure
+        # we passed profile as dict earlier, but gr.State may store as dict -> fine
+        # detect date-like columns heuristically by dtype or name
+        # get columns list
+        cols = list(profile.get("dtypes", {}).keys())
+        date_candidates = [c for c,d in profile.get("dtypes", {}).items() if "datetime" in d or "date" in c.lower()]
+        # fallback: empty
+        return gr.Dropdown.update(choices=cols), gr.Dropdown.update(choices=cols)
+
+    profile_md.change(fn=_populate_prepare, inputs=[profile_state], outputs=[date_col, target_col])
+
+    # apply preparation
+    def _apply_preparation(df_path, impute_num, impute_cat, fill_val, outlier_cols_txt, outlier_method, date_col_sel, date_fmt, text_cols_txt, transform_cols_txt, transform_method, drop_highcard_flag):
+        try:
+            if not df_path:
+                return "Upload dataset first.", pd.DataFrame(), None
+            df = pd.read_csv(df_path)
+            # drop high-card if requested
+            dropped = []
+            if drop_highcard_flag:
+                df, dropped = drop_high_cardinality(df)
+            # parse date
+            if date_col_sel:
+                df = parse_dates(df, date_col_sel, date_fmt if date_fmt else None)
+            # text cleaning
+            if text_cols_txt:
+                tcols = [c.strip() for c in text_cols_txt.split(",") if c.strip()]
+                df = text_clean(df, tcols)
+            # impute
+            df = impute_df(df, numeric_strategy=impute_num, categorical_strategy=impute_cat, fill_value=fill_val)
+            # outliers
+            if outlier_cols_txt:
+                cols = [c.strip() for c in outlier_cols_txt.split(",") if c.strip()]
             else:
-                # hide row & reset
-                updates.append(gr.update(visible=False))
-                updates.append(gr.update(label="", value=False))
-                updates.append(gr.update(value=""))
-                updates.append(gr.update(visible=False, value=1))
-                updates.append(gr.update(choices=[], value=None))
-                updates.append(gr.update(choices=[], value=[]))
-
-        # prepare details html
-        details_html = create_submodule_details_html(comp, mods, submods) if (comp and mods and submods) else ""
-
-        # Return updates plus details_html
-        return updates + [details_html]
-
-    # build outputs mapping (must match the returned updates order)
-    row_outputs = []
-    for i in range(MAX_SUBMODULES):
-        # row, checkbox, name textbox, count, comps_radio, phases_cg
-        row_outputs += [per_sm_row[i], per_sm_checkbox[i], per_sm_name[i], per_sm_count[i], per_sm_complexities[i], per_sm_phases[i]]
-
-    modules.change(
-        fn=on_modules_change,
-        inputs=[competency, modules],
-        outputs=row_outputs + [submodule_details]
-    )
-
-    # ------------------ Calculation wrapper ------------------
-    def collect_and_compute(comp, mods, *args_and_buffer):
-        """
-        args_and_buffer contains: for each row 5 items (chk,name,count,comps,phases) then final argument is buffer_percent
-        We'll return (html_result, hide_inputs_update, show_results_update)
-        """
-        total_expected = MAX_SUBMODULES * 5 + 1
-        if len(args_and_buffer) != total_expected:
-            return ("❌ Internal: unexpected inputs passed to calculator.", gr.update(visible=True), gr.update(visible=False))
-
-        # split
-        row_args = args_and_buffer[:-1]
-        buffer_val = args_and_buffer[-1]
-
-        # parse row args
-        selected_submodules = []
-        submodule_counts = {}
-        per_sub_comps = {}
-        per_sub_phases = {}
-
-        for i in range(MAX_SUBMODULES):
-            base = i * 5
-            chk = row_args[base + 0]
-            name = row_args[base + 1]
-            cnt = row_args[base + 2]
-            comps_sel_raw = row_args[base + 3]   # Radio -> single value or None
-            phases_sel = row_args[base + 4] or []
-
-            # convert complexity single value into a list to keep original logic
-            if comps_sel_raw is None or comps_sel_raw == "":
-                comps_sel = []
+                cols = df.select_dtypes(include=[np.number]).columns.tolist()
+            df = treat_outliers_iqr(df, cols, method=outlier_method)
+            # transform numeric
+            if transform_cols_txt:
+                tcols = [c.strip() for c in transform_cols_txt.split(",") if c.strip()]
+                df = transform_cols(df, tcols, method=transform_method)
+            # save cleaned temp file
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".csv")
+            df.to_csv(tmp.name, index=False)
+            prep_summary = f"Prepared data: {len(df)} rows, {len(df.columns)} columns. Dropped high-card columns: {', '.join(dropped) if dropped else 'None'}"
+            return prep_summary, df.head(200), tmp.name
+        except Exception as e:
+            tb = traceback.format_exc()
+            print(tb)
+            return f"Preparation failed: {e}", pd.DataFrame(), None
+
+    apply_prep.click(fn=_apply_preparation, inputs=[df_state, impute_num, impute_cat, const_fill, outlier_cols, outlier_method, date_col, date_fmt, text_cols, transform_cols_txt, transform_method, drop_highcard], outputs=[prep_output, preview_clean, clean_state])
+
+    download_clean.click(fn=lambda p: p if p else None, inputs=[clean_state], outputs=[download_clean])
+
+    # NLP visualize
+    def _nl_visualize(query, clean_path, df_path):
+        try:
+            df = None
+            if clean_path:
+                df = pd.read_csv(clean_path)
+            elif df_path:
+                df = pd.read_csv(df_path)
+            if df is None or df.empty:
+                return None
+            instr = nlp_to_chart_instruction(query, df)
+            fig = render_chart_from_instruction(instr, df)
+            if fig is None:
+                return None
+            return fig
+        except Exception as e:
+            print("nlp visualize error", e)
+            return None
+
+    nl_btn.click(fn=_nl_visualize, inputs=[nl_input, clean_state, df_state], outputs=[nl_plot])
+
+    # Train model
+    def _train_model(task, target, scaler_choice, model_choice, clean_path, df_path, do_rfecv=False):
+        try:
+            if not target:
+                return "Select target variable.", None, None
+            if clean_path:
+                df = pd.read_csv(clean_path)
+            elif df_path:
+                df = pd.read_csv(df_path)
             else:
-                comps_sel = [comps_sel_raw]
-
-            if chk:
-                if not name:
-                    continue
-                if mods and not any(name in list(sheet1[(sheet1["Competency"] == comp) & (sheet1["Module"] == m)]["Sub - Module"].dropna().unique()) for m in mods):
-                    continue
-                selected_submodules.append(name)
-                try:
-                    submodule_counts[name] = int(cnt) if (cnt is not None and str(cnt).strip() != "") else 1
-                except:
-                    submodule_counts[name] = 1
-                per_sub_comps[name] = comps_sel
-                per_sub_phases[name] = phases_sel
-
-        # call calculation
-        html = calculate_effort_per_submodule(comp, mods, selected_submodules, submodule_counts, per_sub_comps, per_sub_phases, buffer_val)
-        # hide inputs, show results
-        return (html, gr.update(visible=False), gr.update(visible=True))
-
-    # Build inputs list for the calculate button
-    calculation_inputs = [competency, modules]
-    for i in range(MAX_SUBMODULES):
-        calculation_inputs += [per_sm_checkbox[i], per_sm_name[i], per_sm_count[i], per_sm_complexities[i], per_sm_phases[i]]
-    calculation_inputs += [buffer_percent]
-
-    # Clicking Calculate: compute, hide inputs page, show results page, and populate results HTML
-    calculate_btn.click(fn=collect_and_compute, inputs=calculation_inputs, outputs=[results, page_inputs, page_results])
-
-    # Back button returns to inputs, clears results
-    def go_back():
-        return (gr.update(visible=True), gr.update(visible=False), "")
-
-    back_btn.click(fn=go_back, inputs=[], outputs=[page_inputs, page_results, results])
-
-# Launch
-demo.launch(share=True, debug=True)
+                return "No dataset available.", None, None
+            if target not in df.columns:
+                return f"Target '{target}' not in data.", None, None
+            # drop high cardinal categorical columns
+            df2, dropped = drop_high_cardinality(df)
+            X, y = prepare_features_targets(df2, target, drop_cols=None, drop_high_card=False)
+            # auto-encode and scale inside training helper
+            results = fit_and_evaluate_model(X, y, model_choice, task, scaler_choice, do_rfecv=do_rfecv)
+            # create markdown summary
+            md = []
+            md.append(f"## Model: {model_choice}")
+            if task=="regression":
+                md.append(f"- MSE: {results['mse']:.4f}")
+                md.append(f"- RMSE: {results['rmse']:.4f}")
+                md.append(f"- MAE: {results['mae']:.4f}")
+                md.append(f"- R2: {results['r2']:.4f}")
+                # create QQ plot
+                resid = results.get("residuals")
+                fig = plt.figure(figsize=(6,4))
+                stats.probplot(resid.dropna(), dist="norm", plot=plt)
+                plt.title("Q-Q plot of residuals")
+                tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
+                fig.savefig(tmp.name, bbox_inches="tight")
+                plt.close(fig)
+                plot_path = tmp.name
+            else:
+                md.append(f"- Accuracy: {results['accuracy']:.4f}")
+                if results.get("roc_auc") is not None:
+                    md.append(f"- ROC AUC: {results['roc_auc']:.4f}")
+                md.append("### Classification report")
+                cr = results.get("report")
+                if cr:
+                    md.append(pd.DataFrame(cr).to_markdown())
+                # confusion matrix plot
+                cm = results.get("confusion_matrix")
+                fig = go.Figure(data=go.Heatmap(z=cm, x=["pred_"+str(i) for i in range(len(cm))], y=["true_"+str(i) for i in range(len(cm))], colorscale="Blues"))
+                fig.update_layout(title="Confusion Matrix")
+                tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".png")
+                fig.write_image(tmp.name)
+                plot_path = tmp.name
+            # feature importance DataFrame
+            feat_imp = results.get("feature_importance")
+            if feat_imp:
+                fi_df = pd.DataFrame(feat_imp, columns=["feature","importance"])
+            else:
+                fi_df = pd.DataFrame()
+            return ("\n".join(md), fi_df, gr.Image.update(value=plot_path))
+        except Exception as e:
+            tb = traceback.format_exc()
+            print(tb)
+            return f"Training error: {e}", pd.DataFrame(), None
+
+    train_btn.click(fn=_train_model, inputs=[task_select, target_col, scaler_choice, model_select, clean_state, df_state, rfecv_opt], outputs=[model_out_md, model_feature_imp, model_plots])
+
+    # report generation: simple markdown
+    def _generate_report(profile, prep_summary, model_summary, feature_imp_df):
+        try:
+            lines = []
+            lines.append("# Executive Report — DataSynth")
+            lines.append("## Data Profile")
+            if profile:
+                lines.append(profile_to_markdown(profile))
+            else:
+                lines.append("No profile available.")
+            lines.append("\n## Preparation")
+            lines.append(prep_summary or "No preparation performed.")
+            lines.append("\n## Model Summary")
+            lines.append(model_summary or "No model results.")
+            if isinstance(feature_imp_df, pd.DataFrame) and not feature_imp_df.empty:
+                lines.append("\n## Feature Importances")
+                lines.append(feature_imp_df.to_markdown(index=False))
+            # save markdown
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=".md", mode="w", encoding="utf-8")
+            tmp.write("\n".join(lines))
+            tmp.flush()
+            return tmp.name
+        except Exception as e:
+            return None
+
+    report_btn.click(fn=_generate_report, inputs=[profile_state, prep_output, model_out_md, model_feature_imp], outputs=[report_download])
+
+    # initial no-op
+    demo.load(lambda: None, outputs=[])
+
+if __name__ == "__main__":
+    demo.launch(server_name="0.0.0.0", server_port=7860, share=False)