Fresh deployment with LFS tracking

agents/analysis_agent.py

@@ -0,0 +1,329 @@
+from utils.logger import logger
+import numpy as np
+from sklearn.ensemble import RandomForestClassifier
+
+class AnalysisAgent:
+
+    def __init__(self, registry):
+        
+        self.registry = registry
+
+    # ---------------------------
+    # Proper dataset extraction
+    # ---------------------------
+    def _extract_dataset(self, text):
+        print("🔍 Running dataset analysis...")
+        datasets = self.registry.list_datasets()
+        words = str(text).lower().split()
+
+        for word in words:
+            for d in datasets:
+                if word == d.lower():
+                    return d
+        return None
+
+    # ---------------------------
+    # Remove ID-like columns
+    # ---------------------------
+    def _remove_id_like_columns(self, df):
+        cols_to_drop = []
+
+        for col in df.columns:
+            unique_ratio = df[col].nunique() / len(df)
+
+            if unique_ratio > 0.9:
+                cols_to_drop.append(col)
+
+        df_clean = df.drop(columns=cols_to_drop)
+
+        return df_clean, cols_to_drop
+
+    # ---------------------------
+    # Select target column
+    # ---------------------------
+    def _select_target(self, df):
+        candidates = []
+
+        for col in df.columns:
+            unique_count = df[col].nunique()
+            unique_ratio = unique_count / len(df)
+
+            # Skip obvious bad columns
+            if any(k in col.lower() for k in ["id", "name", "email", "phone"]):
+                continue
+
+            #  Skip high-cardinality
+            if unique_ratio > 0.5:
+                continue
+
+            # Prefer categorical / classification targets
+            if unique_count <= 20:
+                candidates.append((col, unique_count))
+
+        #  Pick best candidate (lowest unique count but >1)
+        if candidates:
+            candidates = sorted(candidates, key=lambda x: x[1])
+            return candidates[0][0]
+
+        return None
+
+    # ---------------------------
+    # Feature importance
+    # ---------------------------
+    def _compute_feature_importance(self, df):
+
+        df_clean, dropped_cols = self._remove_id_like_columns(df)
+        if len(df.columns) <= 2:
+            return None, dropped_cols, "Dataset too small for feature importance."
+        target = self._select_target(df_clean)
+        if not target:
+            return None, dropped_cols, "No suitable target column found."
+        
+        y = df_clean[target]
+
+        # Fix NaN issue
+        if y.isnull().sum() > 0:
+            return None, dropped_cols, "Target contains missing values. Cannot compute feature importance."
+        if not target:
+            return None, dropped_cols, "No suitable target column found."
+
+        #  Prevent sklearn warning
+        if df_clean[target].nunique() > 0.5 * len(df_clean):
+            return None, dropped_cols, "Target not suitable for classification."
+
+        X = df_clean.drop(columns=[target])
+        y = df_clean[target]
+
+        # Encode categoricals
+        X = X.apply(lambda col: col.astype('category').cat.codes)
+
+        try:
+            model = RandomForestClassifier(n_estimators=50)
+            model.fit(X, y)
+
+            importances = dict(zip(X.columns, model.feature_importances_))
+            sorted_imp = sorted(importances.items(), key=lambda x: x[1], reverse=True)
+
+            return sorted_imp[:5], dropped_cols, None
+
+        except Exception as e:
+            return None, dropped_cols, str(e)
+
+    # ---------------------------
+    # Optional explanation layer
+    # ---------------------------
+    def _explain_feature(self, col):
+        return f"{col} shows strong predictive signal based on dataset patterns."
+    
+    #----------------------------
+    # Outlier Detection
+    #----------------------------
+    def _detect_outliers(self, df):
+        try:
+            numeric_df = df.select_dtypes(include="number")
+
+            outlier_summary = {}
+
+            for col in numeric_df.columns:
+                q1 = numeric_df[col].quantile(0.25)
+                q3 = numeric_df[col].quantile(0.75)
+                iqr = q3 - q1
+
+                lower = q1 - 1.5 * iqr
+                upper = q3 + 1.5 * iqr
+
+                outliers = numeric_df[(numeric_df[col] < lower) | (numeric_df[col] > upper)]
+
+                if len(outliers) > 0:
+                    outlier_summary[col] = len(outliers)
+
+            return outlier_summary, None
+
+        except Exception as e:
+            logger.error(f"Outlier detection failed | {e}")
+            return None, str(e)
+    
+    #---------------------------
+    # Correlation analysis
+    #---------------------------
+    def _compute_correlation(self, df):
+        try:
+            numeric_df = df.select_dtypes(include="number")
+
+            if numeric_df.shape[1] < 2:
+                return None, "Not enough numeric columns for correlation."
+
+            # corr = numeric_df.corr()
+
+            # Get top correlations (excluding self)
+            corr_matrix = numeric_df.corr().abs()
+
+            upper = corr_matrix.where(
+                np.triu(np.ones(corr_matrix.shape), k=1).astype(bool)
+            )
+
+            top_pairs = (
+                upper.unstack()
+                .dropna()
+                .sort_values(ascending=False)
+                .head(5)
+            )
+            return top_pairs.to_dict(), None
+
+        except Exception as e:
+            logger.error(f"Correlation failed | {e}")
+            return None, str(e)
+        
+    #-------------------------
+    #Saving report    
+    #-------------------------
+    def _export_report(self, dataset, content):
+        try:
+            path = f"output/report_{dataset}.txt"
+
+            with open(path, "w", encoding="utf-8") as f:
+                f.write(content)
+
+            logger.info(f"Report exported: {path}")
+
+            return path
+
+        except Exception as e:
+            logger.error(f"Report export failed | {e}")
+            return None
+
+    # ---------------------------
+    # MAIN HANDLER
+    # ---------------------------
+    def handle(self, dataset=None):
+
+        try:
+            # ---- HANDLE "analyze people" CASE ----
+            if isinstance(dataset, str):
+                extracted = self._extract_dataset(dataset)
+                if extracted:
+                    dataset = extracted
+
+            # ---- STRICT DATASET CHECK ----
+            if not dataset:
+                return "Please specify a dataset (e.g., 'analyze people')"
+
+            df = self.registry.load_dataframe(dataset)
+
+        except Exception as e:
+            logger.error(f"Failed loading dataset | {e}")
+            return f"Failed to load dataset: {dataset}"
+
+        try:
+            # ---------- OUTPUT ----------
+            output = []
+            rows, cols = df.shape
+            print("🧹 Checking duplicates...")
+            # ---------- DATA QUALITY ----------
+            total_missing = df.isnull().sum().sum()
+            duplicates = df.duplicated().sum()
+
+            missing_by_column = df.isnull().sum()
+            missing_by_column = missing_by_column[missing_by_column > 0]
+
+            # ---------- COLUMN TYPES ----------
+            numeric_cols = df.select_dtypes(include="number").columns.tolist()
+            categorical_cols = df.select_dtypes(exclude="number").columns.tolist()
+
+            # ---------- WARNINGS ----------
+            print("⚠️ Generating warnings...")
+            warnings = []
+
+            for col in df.columns:
+                if len(df) == 0:
+                    continue
+
+                unique_ratio = df[col].nunique() / len(df)
+
+                if unique_ratio > 0.95 and "id" in col.lower():
+                    warnings.append(f"{col} looks like an ID column")
+
+                missing_ratio = df[col].isnull().sum() / len(df)
+                if missing_ratio > 0.5:
+                    warnings.append(f"{col} has {missing_ratio:.2%} missing values")
+
+                if df[col].nunique() == 1:
+                    warnings.append(f"{col} is constant (no variance)")
+
+            # ---------- FEATURE IMPORTANCE (NEW CLEAN VERSION) ----------
+            print("📈 Looking for potential feature importance...")
+            fi, dropped_cols, error = self._compute_feature_importance(df)
+
+            # ---------- CORRELATION ANALYSIS ----------
+            print("📊 Computing correlation...")
+            corr_pairs, corr_error = self._compute_correlation(df)
+
+            output= []
+
+            output.append(f"\nDataset Analysis: {dataset}")
+            output.append("=" * 40)
+
+            output.append(f"Rows: {rows}")
+            output.append(f"Columns: {cols}")
+
+            output.append("\nData Quality")
+            output.append("-" * 20)
+            output.append(f"Total Missing Values : {total_missing}")
+            output.append(f"Duplicate Rows       : {duplicates}")
+            # ---------- CORRELATION OUTPUT ----------
+            output.append("\nTop Correlations")
+            output.append("-" * 20)
+
+
+            if corr_error:
+                output.append(corr_error)
+            elif corr_pairs is not None:
+                for (col1, col2), val in corr_pairs.items():
+                    output.append(f"{col1} ↔ {col2}: {val:.3f}")
+            else:
+                output.append("No correlation data available.")
+            if not missing_by_column.empty:
+                output.append("\nMissing by Column")
+                output.append("-" * 20)
+                for col, val in missing_by_column.items():
+                    output.append(f"{col}: {val}")
+
+            output.append("\nColumn Types")
+            output.append("-" * 20)
+            output.append(f"Numeric      : {', '.join(numeric_cols) if numeric_cols else 'None'}")
+            output.append(f"Categorical  : {', '.join(categorical_cols) if categorical_cols else 'None'}")
+
+            if warnings:
+                output.append("\n⚠️ Data Warnings")
+                output.append("-" * 20)
+                for w in warnings[:5]:
+                    output.append(f"- {w}")
+
+            # ---------- FEATURE IMPORTANCE OUTPUT ----------
+            output.append("\nPotential Feature Importance")
+            output.append("-" * 20)
+
+            if error:
+                output.append(error)
+            else:
+                for col, score in fi:
+                    explanation = self._explain_feature(col)
+                    output.append(f"{col}: {score:.4f} → {explanation}")
+
+            # ---------- DROPPED COLUMNS ----------
+            if dropped_cols:
+                output.append("\n⚠️ Ignored high-cardinality columns:")
+                for col in dropped_cols:
+                    output.append(f"- {col}")
+
+             # ---------- EXPORT (ONLY ONCE) ----------
+            report_path = self._export_report(dataset, "\n".join(output))
+
+            if report_path:
+                output.append(f"\n📁 Report saved to: {report_path}")
+
+            return "\n".join(output)
+
+        except Exception as e:
+            logger.error(f"Analysis failed | {e}")
+            return "Analysis agent error."

agents/dataframe_agent.py

@@ -0,0 +1,141 @@
+from utils.logger import logger
+
+
+class DataFrameAgent:
+
+    def __init__(self, registry):
+        self.registry = registry
+
+
+    def _detect_dataset(self, query, datasets):
+        """
+        Detect dataset name from query.
+        Falls back to first dataset if none mentioned.
+        """
+        q = query.lower()
+
+        for d in datasets:
+            if d.lower() in q:
+                return d
+
+        logger.info("Dataset not specified, using default dataset.")
+        return datasets[0]
+
+
+    def _detect_column(self, query, columns):
+        """
+        Detect column name from query.
+        """
+        q = query.lower()
+
+        for col in columns:
+            if col.lower() in q:
+                return col
+
+        return None
+
+
+    def _detect_number(self, query, default=5):
+        """
+        Extract number from query (used for top N rows).
+        """
+        words = query.split()
+
+        for w in words:
+            if w.isdigit():
+                return int(w)
+
+        return default
+
+
+    def handle(self, query):
+
+        q = query.lower()
+
+        try:
+
+            datasets = self.registry.list_datasets()
+
+            if not datasets:
+                logger.warning("DataFrameAgent called with no datasets loaded.")
+                return "No datasets available."
+
+            dataset = self._detect_dataset(q, datasets)
+
+            df = self.registry.load_dataframe(dataset)
+
+            columns = df.columns.tolist()
+
+        except Exception as e:
+            logger.error(f"Failed loading dataset in DataFrameAgent | {e}")
+            return "Failed to load dataset."
+
+
+        try:
+
+            # -------- SHOW ROWS --------
+            if "top" in q or "first" in q:
+
+                n = self._detect_number(q, default=5)
+
+                logger.info(f"Showing first {n} rows from {dataset}")
+
+                return df.head(n)
+
+
+            # -------- ROW COUNT --------
+            if "how many rows" in q or "row count" in q or "count rows" in q:
+
+                logger.info(f"Row count requested for {dataset}")
+
+                return f"{dataset} has {len(df)} rows."
+
+
+            # -------- COLUMN DETECTION --------
+            column = self._detect_column(q, columns)
+
+            if column is None and any(
+                word in q for word in ["average", "mean", "max", "min", "highest", "lowest"]
+            ):
+                logger.warning("Column not detected for dataframe operation.")
+                return "Column not found in dataset."
+
+
+            # -------- MEAN / AVERAGE --------
+            if "average" in q or "mean" in q:
+
+                result = df[column].mean()
+
+                logger.info(f"Mean computed for {column} in {dataset}")
+
+                return f"Average {column} in {dataset}: {round(result, 2)}"
+
+
+            # -------- MAX --------
+            if "max" in q or "highest" in q:
+
+                result = df[column].max()
+
+                logger.info(f"Max computed for {column} in {dataset}")
+
+                return f"Max {column} in {dataset}: {result}"
+
+
+            # -------- MIN --------
+            if "min" in q or "lowest" in q:
+
+                result = df[column].min()
+
+                logger.info(f"Min computed for {column} in {dataset}")
+
+                return f"Min {column} in {dataset}: {result}"
+
+
+            return "DataFrame query not understood."
+
+
+        except Exception as e:
+
+            logger.error(f"DataFrame operation failed | Query: {query} | Error: {e}")
+
+            return "DataFrame agent error."

agents/metadata_agent.py

@@ -0,0 +1,73 @@
+from utils.logger import logger
+
+
+class MetadataAgent:
+
+    def __init__(self, registry):
+        self.registry = registry
+
+
+    def _detect_dataset(self, query, datasets):
+
+        q = query.lower()
+
+        for d in datasets:
+            if d.lower() in q:
+                return d
+
+        # fallback to first dataset
+        return datasets[0]
+
+
+    def handle(self, query):
+
+        q = query.lower()
+
+        try:
+
+            datasets = self.registry.list_datasets()
+
+            if not datasets:
+                return "No datasets available."
+
+            dataset = self._detect_dataset(q, datasets)
+
+            meta = self.registry.get_info(dataset)
+
+            cols = meta.get("columns", [])
+            nums = meta.get("numeric_columns", [])
+            cats = meta.get("categorical_columns", [])
+            miss = meta.get("missing_values", {})
+
+            # ---- INTENT DETECTION ----
+
+            if "how many column" in q or "number of column" in q:
+                return f"{dataset} has {len(cols)} columns."
+
+            if "numeric" in q:
+                if not nums:
+                    return f"No numeric columns found in {dataset}."
+                return f"Numeric columns in {dataset}: {', '.join(nums)}"
+
+            if "categorical" in q:
+                if not cats:
+                    return f"No categorical columns found in {dataset}."
+                return f"Categorical columns in {dataset}: {', '.join(cats)}"
+
+            if "missing" in q:
+                if not miss:
+                    return f"No missing value info for {dataset}."
+                return f"Missing values in {dataset}: {miss}"
+
+            if "column" in q:
+                if not cols:
+                    return f"No columns found for {dataset}."
+                return f"Columns in {dataset}: {', '.join(cols)}"
+            logger.info(f"MetadataAgent | dataset={dataset} | query={query}")
+            return "Metadata query not understood."
+
+        except Exception as e:
+
+            logger.error(f"Metadata agent failed | {e}")
+
+            return "Metadata agent error."

agents/transformer_agent.py

@@ -0,0 +1,518 @@
+import pandas as pd
+from data.schema_extractor import extract_schema
+from utils.logger import logger
+
+
+NUMERIC_TYPES = ("int64", "float64", "int32", "float32")
+CATEGORICAL_TYPES = ("object", "category", "str", "string")
+
+
+class TransformerAgent:
+    """
+    All operations target a '<dataset>_clean' copy of the original.
+    The source dataset is never modified.
+
+    Supports plan-based dispatch (from LLM) and keyword-based fallback.
+
+    Cleaning:
+        drop_duplicates, drop_column, drop_constant_columns,
+        strip_whitespace, drop_missing_rows, drop_missing_cols
+
+    Filling:
+        fill_nulls  — smart auto (mean/median for numeric, mode for categorical)
+        fill_mean   — explicit mean
+        fill_median — explicit median
+        fill_mode   — explicit mode
+        fill_zero   — fill with 0 / empty string
+
+    Scaling / Encoding:
+        normalize   — min-max [0, 1]
+        standardize — z-score (subtract mean, divide by std)
+        encode      — label encoding (int codes)
+        onehot      — one-hot encoding (pd.get_dummies, ≤20 unique values)
+
+    Other:
+        rename      — rename <old> to <new> in <dataset>
+    """
+
+    def __init__(self, registry):
+        self.registry = registry
+
+    # ── helpers ────────────────────────────────────────────────────────────
+
+    def _detect_source(self, query, datasets):
+        q = query.lower()
+        base = [d for d in datasets if not d.endswith("_clean")]
+        for d in base:
+            if d.lower() in q:
+                return d
+        return base[0] if base else datasets[0]
+
+    def _detect_column(self, query, columns):
+        q = query.lower()
+        for col in columns:
+            if col.lower() in q:
+                return col
+        return None
+
+    def _resolve_column(self, plan_col, query, columns):
+        """
+        Return the real column name, checking the plan first then falling
+        back to keyword scan. Returns None if nothing found.
+        """
+        if plan_col:
+            for col in columns:
+                if col.lower() == plan_col.lower():
+                    return col
+        return self._detect_column(query, columns)
+
+    def _get_working_dataset(self, source_name):
+        clean_name = f"{source_name}_clean"
+        datasets = self.registry.list_datasets()
+        if clean_name not in datasets:
+            source_df = self.registry.load_dataframe(source_name)
+            schema = extract_schema(source_df)
+            self.registry.register_dataset(clean_name, source_df.copy(), schema)
+            logger.info(f"Created clean copy: {clean_name}")
+        df = self.registry.load_dataframe(clean_name)
+        return clean_name, df
+
+    def _save(self, name, df):
+        self.registry.update_dataset(name, df, extract_schema(df))
+
+    def _smart_fill_column(self, series):
+        """Auto pick mean/median for numeric, mode for categorical."""
+        if series.dtype.name in NUMERIC_TYPES:
+            skewness = abs(series.skew())
+            if skewness < 1:
+                val = series.mean()
+                return series.fillna(val), f"mean ({round(val, 4)})"
+            val = series.median()
+            return series.fillna(val), f"median ({round(val, 4)})"
+        mode_val = series.mode()
+        if mode_val.empty:
+            return series, "no mode found"
+        val = mode_val[0]
+        return series.fillna(val), f"mode ('{val}')"
+
+    # ── individual operation methods ───────────────────────────────────────
+
+    def _op_drop_duplicates(self, clean_name, df):
+        before = len(df)
+        str_cols = [c for c in df.columns if df[c].dtype.name in ("str", "string", "object")]
+        df[str_cols] = df[str_cols].astype(object)
+        df = df.drop_duplicates()
+        removed = before - len(df)
+        self._save(clean_name, df)
+        logger.info(f"Dropped {removed} duplicates from {clean_name}")
+        return (
+            f"Dropped {removed} duplicate row(s) from '{clean_name}'. "
+            f"Rows remaining: {len(df)}."
+        )
+
+    def _op_drop_constant_columns(self, clean_name, df):
+        const_cols = [c for c in df.columns if df[c].nunique() <= 1]
+        if not const_cols:
+            return f"No constant columns found in '{clean_name}'."
+        df = df.drop(columns=const_cols)
+        self._save(clean_name, df)
+        logger.info(f"Dropped constant columns {const_cols} from {clean_name}")
+        return (
+            f"Dropped {len(const_cols)} constant column(s) from '{clean_name}': "
+            f"{', '.join(const_cols)}."
+        )
+
+    def _op_strip_whitespace(self, clean_name, df):
+        str_cols = [c for c in df.columns if df[c].dtype.name in ("object", "str", "string")]
+        if not str_cols:
+            return f"No string columns to strip in '{clean_name}'."
+        for col in str_cols:
+            df[col] = df[col].str.strip()
+        self._save(clean_name, df)
+        logger.info(f"Stripped whitespace in {len(str_cols)} columns in {clean_name}")
+        return (
+            f"Stripped leading/trailing whitespace from {len(str_cols)} "
+            f"string column(s) in '{clean_name}'."
+        )
+
+    def _op_drop_missing_rows(self, clean_name, df):
+        before = len(df)
+        df = df.dropna()
+        removed = before - len(df)
+        self._save(clean_name, df)
+        logger.info(f"Dropped {removed} rows with nulls from {clean_name}")
+        return (
+            f"Dropped {removed} row(s) containing missing values from '{clean_name}'. "
+            f"Rows remaining: {len(df)}."
+        )
+
+    def _op_drop_missing_cols(self, clean_name, df, threshold=0.5):
+        missing_pct = df.isnull().mean()
+        drop_cols = missing_pct[missing_pct > threshold].index.tolist()
+        if not drop_cols:
+            return f"No columns exceed {int(threshold * 100)}% missing threshold in '{clean_name}'."
+        df = df.drop(columns=drop_cols)
+        self._save(clean_name, df)
+        logger.info(f"Dropped high-null columns {drop_cols} from {clean_name}")
+        return (
+            f"Dropped {len(drop_cols)} column(s) with >{int(threshold * 100)}% missing "
+            f"from '{clean_name}': {', '.join(drop_cols)}."
+        )
+
+    def _op_drop_column(self, clean_name, df, column):
+        if column is None:
+            return "Column not found in dataset."
+        df = df.drop(columns=[column])
+        self._save(clean_name, df)
+        logger.info(f"Dropped column '{column}' from {clean_name}")
+        return f"Column '{column}' dropped from '{clean_name}'."
+
+    def _op_fill_smart(self, clean_name, df, column):
+        """Smart fill — auto selects mean/median/mode."""
+        if column is not None:
+            nulls = df[column].isnull().sum()
+            if nulls == 0:
+                return f"Column '{column}' in '{clean_name}' has no missing values."
+            df[column], label = self._smart_fill_column(df[column])
+            self._save(clean_name, df)
+            logger.info(f"Smart-filled {nulls} nulls in '{column}' in {clean_name}")
+            return f"Filled {nulls} missing value(s) in '{column}' using {label}."
+
+        report = []
+        for col in df.columns:
+            nulls = df[col].isnull().sum()
+            if nulls > 0:
+                df[col], label = self._smart_fill_column(df[col])
+                report.append(f"  '{col}': {nulls} filled using {label}")
+        if not report:
+            return f"No missing values found in '{clean_name}'."
+        self._save(clean_name, df)
+        logger.info(f"Smart-filled all nulls in {clean_name}")
+        return "Filled missing values:\n" + "\n".join(report)
+
+    def _op_fill_with(self, clean_name, df, column, strategy):
+        """
+        Fill with an explicit strategy: mean | median | mode | zero.
+        If column is None, applies to all eligible columns.
+        """
+        if strategy == "zero":
+            targets = [column] if column else df.columns.tolist()
+            report = []
+            for col in targets:
+                nulls = df[col].isnull().sum()
+                if nulls == 0:
+                    continue
+                fill_val = 0 if df[col].dtype.name in NUMERIC_TYPES else ""
+                df[col] = df[col].fillna(fill_val)
+                report.append(f"  '{col}': {nulls} filled with {fill_val!r}")
+            if not report:
+                return f"No missing values found in '{clean_name}'."
+            self._save(clean_name, df)
+            return f"Filled with zero:\n" + "\n".join(report)
+
+        if strategy in ("mean", "median"):
+            targets = (
+                [column] if column
+                else [c for c in df.columns if df[c].dtype.name in NUMERIC_TYPES]
+            )
+            if not targets:
+                return "No numeric columns available for this operation."
+            report = []
+            for col in targets:
+                nulls = df[col].isnull().sum()
+                if nulls == 0:
+                    continue
+                if df[col].dtype.name not in NUMERIC_TYPES:
+                    report.append(f"  '{col}': skipped (not numeric)")
+                    continue
+                val = df[col].mean() if strategy == "mean" else df[col].median()
+                df[col] = df[col].fillna(val)
+                report.append(f"  '{col}': {nulls} filled with {strategy} ({round(val, 4)})")
+            if not report:
+                return f"No missing values in numeric columns of '{clean_name}'."
+            self._save(clean_name, df)
+            return f"Filled with {strategy}:\n" + "\n".join(report)
+
+        if strategy == "mode":
+            targets = [column] if column else df.columns.tolist()
+            report = []
+            for col in targets:
+                nulls = df[col].isnull().sum()
+                if nulls == 0:
+                    continue
+                mode_val = df[col].mode()
+                if mode_val.empty:
+                    report.append(f"  '{col}': skipped (no mode)")
+                    continue
+                df[col] = df[col].fillna(mode_val[0])
+                report.append(f"  '{col}': {nulls} filled with mode ('{mode_val[0]}')")
+            if not report:
+                return f"No missing values found in '{clean_name}'."
+            self._save(clean_name, df)
+            return f"Filled with mode:\n" + "\n".join(report)
+
+        return f"Unknown fill strategy: {strategy!r}"
+
+    def _op_normalize(self, clean_name, df, column, columns):
+        """Min-max normalization to [0, 1]."""
+        targets = (
+            [column] if column
+            else [c for c in columns if df[c].dtype.name in NUMERIC_TYPES]
+        )
+        if not targets:
+            return "No numeric columns to normalize."
+        report = []
+        for col in targets:
+            if df[col].dtype.name not in NUMERIC_TYPES:
+                report.append(f"  '{col}': skipped (not numeric)")
+                continue
+            col_min, col_max = df[col].min(), df[col].max()
+            if col_max == col_min:
+                report.append(f"  '{col}': skipped (constant value)")
+                continue
+            df[col] = (df[col] - col_min) / (col_max - col_min)
+            report.append(f"  '{col}': normalized to [0, 1]")
+        if not report:
+            return f"No columns were normalized in '{clean_name}'."
+        self._save(clean_name, df)
+        logger.info(f"Normalized columns in {clean_name}")
+        return f"Min-max normalization applied in '{clean_name}':\n" + "\n".join(report)
+
+    def _op_standardize(self, clean_name, df, column, columns):
+        """Z-score standardization: (x - mean) / std."""
+        targets = (
+            [column] if column
+            else [c for c in columns if df[c].dtype.name in NUMERIC_TYPES]
+        )
+        if not targets:
+            return "No numeric columns to standardize."
+        report = []
+        for col in targets:
+            if df[col].dtype.name not in NUMERIC_TYPES:
+                report.append(f"  '{col}': skipped (not numeric)")
+                continue
+            std = df[col].std()
+            if std == 0:
+                report.append(f"  '{col}': skipped (zero variance)")
+                continue
+            mean = df[col].mean()
+            df[col] = (df[col] - mean) / std
+            report.append(
+                f"  '{col}': standardized (mean={round(mean, 4)}, std={round(std, 4)})"
+            )
+        if not report:
+            return f"No columns were standardized in '{clean_name}'."
+        self._save(clean_name, df)
+        logger.info(f"Standardized columns in {clean_name}")
+        return f"Z-score standardization applied in '{clean_name}':\n" + "\n".join(report)
+
+    def _op_encode(self, clean_name, df, column):
+        """Label encoding — categorical → integer codes."""
+        if column is None:
+            return "Specify a column to encode."
+        if df[column].dtype.name not in CATEGORICAL_TYPES:
+            return f"Column '{column}' is not categorical. Cannot label-encode."
+        categories = df[column].astype("category").cat.categories.tolist()
+        df[column] = df[column].astype("category").cat.codes
+        self._save(clean_name, df)
+        logger.info(f"Label-encoded '{column}' in {clean_name}")
+        return (
+            f"Column '{column}' in '{clean_name}' label-encoded. "
+            f"Categories: {categories[:10]}"
+            f"{'...' if len(categories) > 10 else ''}"
+        )
+
+    def _op_onehot(self, clean_name, df, column, columns, max_unique=20):
+        """One-hot encoding via pd.get_dummies (≤max_unique unique values)."""
+        targets = (
+            [column] if column
+            else [c for c in columns if df[c].dtype.name in CATEGORICAL_TYPES]
+        )
+        if not targets:
+            return "No categorical columns for one-hot encoding."
+        report = []
+        for col in targets:
+            if df[col].dtype.name not in CATEGORICAL_TYPES:
+                report.append(f"  '{col}': skipped (not categorical)")
+                continue
+            unique_n = df[col].nunique()
+            if unique_n > max_unique:
+                report.append(
+                    f"  '{col}': skipped ({unique_n} unique values exceeds limit of {max_unique})"
+                )
+                continue
+            dummies = pd.get_dummies(df[col], prefix=col, drop_first=False)
+            df = df.drop(columns=[col])
+            df = pd.concat([df, dummies], axis=1)
+            report.append(f"  '{col}': expanded into {len(dummies.columns)} columns")
+        if not report:
+            return f"No columns were one-hot encoded in '{clean_name}'."
+        self._save(clean_name, df)
+        logger.info(f"One-hot encoded columns in {clean_name}")
+        return f"One-hot encoding applied in '{clean_name}':\n" + "\n".join(report)
+
+    def _op_rename(self, clean_name, df, columns, query):
+        q = query.lower()
+        try:
+            after = q.split("rename", 1)[1]
+            parts = after.split(" to ", 1)
+            old_raw = parts[0].strip()
+            new_raw = parts[1].strip().split()[0]
+            old_name = next((c for c in columns if c.lower() == old_raw), None)
+            if old_name is None:
+                return f"Column '{old_raw}' not found in dataset."
+            df = df.rename(columns={old_name: new_raw})
+            self._save(clean_name, df)
+            logger.info(f"Renamed '{old_name}' → '{new_raw}' in {clean_name}")
+            return f"Column '{old_name}' renamed to '{new_raw}' in '{clean_name}'."
+        except Exception:
+            return "Could not parse rename. Use: rename <old> to <new> in <dataset>"
+
+    # ── plan-based dispatch ────────────────────────────────────────────────
+
+    def _dispatch_plan(self, plan, clean_name, df, columns):
+        """
+        Directly execute the operation named in the LLM plan.
+        Bypasses keyword matching for precise execution.
+        """
+        op     = plan.get("operation")
+        p_col  = plan.get("column")
+        column = self._resolve_column(p_col, "", columns)
+
+        dispatch = {
+            "drop_duplicates":      lambda: self._op_drop_duplicates(clean_name, df),
+            "drop_constant_columns":lambda: self._op_drop_constant_columns(clean_name, df),
+            "strip_whitespace":     lambda: self._op_strip_whitespace(clean_name, df),
+            "drop_missing_rows":    lambda: self._op_drop_missing_rows(clean_name, df),
+            "drop_missing_cols":    lambda: self._op_drop_missing_cols(clean_name, df),
+            "drop_column":          lambda: self._op_drop_column(clean_name, df, column),
+            "fill_nulls":           lambda: self._op_fill_smart(clean_name, df, column),
+            "fill_mean":            lambda: self._op_fill_with(clean_name, df, column, "mean"),
+            "fill_median":          lambda: self._op_fill_with(clean_name, df, column, "median"),
+            "fill_mode":            lambda: self._op_fill_with(clean_name, df, column, "mode"),
+            "fill_zero":            lambda: self._op_fill_with(clean_name, df, column, "zero"),
+            "normalize":            lambda: self._op_normalize(clean_name, df, column, columns),
+            "standardize":          lambda: self._op_standardize(clean_name, df, column, columns),
+            "encode":               lambda: self._op_encode(clean_name, df, column),
+            "onehot":               lambda: self._op_onehot(clean_name, df, column, columns),
+            "rename":               lambda: self._op_rename(clean_name, df, columns, ""),
+        }
+
+        fn = dispatch.get(op)
+        if fn:
+            logger.info(f"Plan dispatch | op={op} | col={column} | dataset={clean_name}")
+            return fn()
+        return (
+            f"Operation '{op}' is not implemented in the transformer agent."
+        )
+
+    # ── keyword-based fallback ─────────────────────────────────────────────
+
+    def _dispatch_keywords(self, q, query, clean_name, df, columns):
+        """Keyword-based routing used when no LLM plan is available."""
+
+        # ── CLEANING ────────────────────────────────────────────────────
+
+        if "duplicate" in q:
+            return self._op_drop_duplicates(clean_name, df)
+
+        if "constant" in q:
+            return self._op_drop_constant_columns(clean_name, df)
+
+        if "strip" in q or "whitespace" in q:
+            return self._op_strip_whitespace(clean_name, df)
+
+        if "drop missing row" in q or "drop na" in q or "dropna" in q:
+            return self._op_drop_missing_rows(clean_name, df)
+
+        if "drop missing col" in q:
+            return self._op_drop_missing_cols(clean_name, df)
+
+        # ── FILLING ─────────────────────────────────────────────────────
+
+        if "fill" in q or "impute" in q:
+            column = self._detect_column(q, columns)
+
+            # Explicit strategy keywords take priority over smart fill
+            if "mean" in q:
+                return self._op_fill_with(clean_name, df, column, "mean")
+            if "median" in q:
+                return self._op_fill_with(clean_name, df, column, "median")
+            if "mode" in q:
+                return self._op_fill_with(clean_name, df, column, "mode")
+            if "zero" in q or " 0 " in q:
+                return self._op_fill_with(clean_name, df, column, "zero")
+
+            # Default: smart auto-fill
+            return self._op_fill_smart(clean_name, df, column)
+
+        # ── DROP COLUMN ─────────────────────────────────────────────────
+
+        if "drop" in q:
+            column = self._detect_column(q, columns)
+            return self._op_drop_column(clean_name, df, column)
+
+        # ── TRANSFORMS ──────────────────────────────────────────────────
+
+        if "standardize" in q or "zscore" in q or "z-score" in q:
+            column = self._detect_column(q, columns)
+            return self._op_standardize(clean_name, df, column, columns)
+
+        if "normalize" in q or "scale" in q:
+            column = self._detect_column(q, columns)
+            return self._op_normalize(clean_name, df, column, columns)
+
+        if "one hot" in q or "onehot" in q or "one-hot" in q or "dummies" in q:
+            column = self._detect_column(q, columns)
+            return self._op_onehot(clean_name, df, column, columns)
+
+        if "encode" in q:
+            column = self._detect_column(q, columns)
+            return self._op_encode(clean_name, df, column)
+
+        if "rename" in q and " to " in q:
+            return self._op_rename(clean_name, df, columns, query)
+
+        return (
+            "Operation not understood. Supported — "
+            "cleaning: drop duplicates, drop column, drop constant columns, "
+            "strip whitespace, drop missing rows, drop missing cols; "
+            "filling: fill nulls / fill with mean / median / mode / zero; "
+            "scaling: normalize, standardize; "
+            "encoding: encode (label), onehot; "
+            "other: rename."
+        )
+
+    # ── public entry point ─────────────────────────────────────────────────
+
+    def handle(self, query, plan=None):
+        q = query.lower()
+
+        try:
+            all_datasets = self.registry.list_datasets()
+            if not all_datasets:
+                logger.warning("TransformerAgent called with no datasets loaded.")
+                return "No datasets available."
+
+            # Dataset resolution: prefer plan's dataset, fall back to keyword scan
+            if plan and plan.get("dataset"):
+                raw_ds = plan["dataset"].replace("_clean", "")
+                source = raw_ds if raw_ds in all_datasets else self._detect_source(q, all_datasets)
+            else:
+                source = self._detect_source(q, all_datasets)
+
+            clean_name, df = self._get_working_dataset(source)
+            columns = df.columns.tolist()
+
+        except Exception as e:
+            logger.error(f"TransformerAgent failed to load dataset | {e}")
+            return "Failed to load dataset."
+
+        try:
+            if plan and plan.get("operation"):
+                return self._dispatch_plan(plan, clean_name, df, columns)
+            return self._dispatch_keywords(q, query, clean_name, df, columns)
+
+        except Exception as e:
+            logger.error(f"TransformerAgent error | Query: {query} | {e}")
+            return "Transformer agent error."

agents/visualization_agent.py

@@ -0,0 +1,145 @@
+from utils.logger import logger
+import matplotlib.pyplot as plt
+import plotext as plt_terminal
+import os
+
+
+class VisualizationAgent:
+
+    def __init__(self, registry):
+        self.registry = registry
+        os.makedirs("output", exist_ok=True)
+
+
+    def _detect_dataset(self, query, datasets):
+
+        q = query.lower()
+
+        for d in datasets:
+            if d.lower() in q:
+                return d
+
+        logger.info("Dataset not specified, using default dataset.")
+        return datasets[0]
+
+
+    def _detect_column(self, query, columns):
+
+        q = query.lower()
+
+        for col in columns:
+            if col.lower() in q:
+                return col
+
+        return None
+
+
+    def handle(self, query):
+
+        q = query.lower()
+
+        try:
+
+            datasets = self.registry.list_datasets()
+
+            if not datasets:
+                logger.warning("VisualizationAgent called with no datasets loaded.")
+                return "No datasets available."
+
+            dataset = self._detect_dataset(q, datasets)
+
+            df = self.registry.load_dataframe(dataset)
+
+            columns = df.columns.tolist()
+
+        except Exception as e:
+
+            logger.error(f"Failed loading dataset in VisualizationAgent | {e}")
+            return "Failed to load dataset."
+
+        try:
+
+            column = self._detect_column(q, columns)
+
+            if column is None:
+                logger.warning("Column not detected for visualization.")
+                return "Column not found in dataset."
+
+            # ---------- HISTOGRAM ----------
+            if "hist" in q or "histogram" in q:
+
+                logger.info(f"Generating histogram for {column} in {dataset}")
+
+                values = df[column].dropna().values
+
+                # Terminal plot
+                plt_terminal.clear_figure()
+                plt_terminal.hist(values, bins=20)
+                plt_terminal.title(f"Histogram of {column}")
+                plt_terminal.xlabel(column)
+                plt_terminal.ylabel("Frequency")
+                plt_terminal.show()
+
+                # Save PNG
+                filepath = f"output/{dataset}_{column}_hist.png"
+
+                plt.figure()
+                df[column].dropna().hist()
+                plt.title(f"Histogram of {column}")
+                plt.xlabel(column)
+                plt.ylabel("Frequency")
+                plt.savefig(filepath)
+                plt.close()
+
+                logger.info(f"Histogram saved → {filepath}")
+
+                return f"Histogram generated in terminal. PNG saved to {filepath}"
+
+
+            # ---------- BAR CHART ----------
+            if "bar" in q or "bar chart" in q:
+
+                unique_values = df[column].nunique()
+
+                if unique_values > 50:
+                    logger.warning(
+                        f"Column '{column}' has {unique_values} unique values. Skipping bar chart."
+                    )
+                    return f"Column '{column}' has {unique_values} unique values. Too many to visualize meaningfully."
+
+                logger.info(f"Generating bar chart for {column} in {dataset}")
+
+                counts = df[column].value_counts()
+
+                # Terminal plot
+                plt_terminal.clear_figure()
+                plt_terminal.bar(
+                    counts.index.astype(str).tolist(),
+                    counts.values.tolist()
+                )
+                plt_terminal.title(f"Bar Chart of {column}")
+                plt_terminal.xlabel(column)
+                plt_terminal.ylabel("Count")
+                plt_terminal.show()
+
+                # Save PNG
+                filepath = f"output/{dataset}_{column}_bar.png"
+
+                plt.figure()
+                counts.plot(kind="bar")
+                plt.title(f"Bar Chart of {column}")
+                plt.xlabel(column)
+                plt.ylabel("Count")
+                plt.savefig(filepath)
+                plt.close()
+
+                logger.info(f"Bar chart saved → {filepath}")
+
+                return f"Bar chart generated in terminal. PNG saved to {filepath}"
+
+            return "Visualization query not understood."
+
+        except Exception as e:
+
+            logger.error(f"Visualization failed | Query: {query} | Error: {e}")
+            return "Visualization agent error."

attached_assets/Pasted-list-me-all-the-datasets-i-have-Datasets-Name--17735902_1773590289755.txt

@@ -0,0 +1,51 @@
+list me all the datasets i have 
+    Datasets     
+┏━━━━━━━━━━━━━━━┓
+┃ Name          ┃
+┡━━━━━━━━━━━━━━━┩
+│ leads         │
+│ organizations │
+│ people        │
+└───────────────┘
+> tell me the no of datasets i have 
+Unknown command. Type 'help' to see available commands.
+> how many datasets do i have
+Unknown command. Type 'help' to see available commands.
+> clean 
+Unknown command. Type 'help' to see available commands.
+> how many columsn are present in leads dataset 
+Unknown command. Type 'help' to see available commands.
+> columns in leads
+Error: Dataset not found
+> visualize people dataset with historgram 
+Column not found in dataset.
+> people info 
+Unknown command. Type 'help' to see available commands.
+> info people
+                                      Dataset Info: people                                      
+┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+┃ Property            ┃ Value                                                                  ┃
+┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
+│ Rows                │ 10000                                                                  │
+│ Columns             │ 9                                                                      │
+│ Numeric Columns     │ Index                                                                  │
+│ Categorical Columns │ User Id, First Name, Last Name, Sex, Email, Phone, Date of birth, Job  │
+│                     │ Title                                                                  │
+│ Column Types        │ Index:int64, User Id:str, First Name:str, Last Name:str, Sex:str,      │
+│                     │ Email:str, Phone:str, Date of birth:str, Job Title:str                 │
+└─────────────────────┴────────────────────────────────────────────────────────────────────────┘
+> describe people
+          Index
+count  10000.00
+mean    5000.50
+std     2886.90
+min        1.00
+25%     2500.75
+50%     5000.50
+75%     7500.25
+max    10000.00
+> fill the skewed columns in people dataset 
+No missing values found in 'people_clean'.
+> when was the leads dataset created and how many records exists 
+Unknown command. Type 'help' to see available commands.
+> ^C

cli_app/command_handler.py

@@ -0,0 +1,324 @@
+from rich.table import Table
+from rich.console import Console
+from data.loader import load_dataset
+from data.schema_extractor import extract_schema
+from data.registry import DatasetRegistry
+from utils.logger import logger
+from core.query_router import QueryRouter
+from agents.metadata_agent import MetadataAgent
+from agents.dataframe_agent import DataFrameAgent
+from agents.visualization_agent import VisualizationAgent
+from agents.transformer_agent import TransformerAgent
+from core.llm_planner import LLMPlanner
+from agents.analysis_agent import AnalysisAgent
+from data.registry import DatasetRegistry
+router = QueryRouter()
+llm_planner = LLMPlanner()
+console = Console()
+registry = DatasetRegistry()
+metadata_agent = MetadataAgent(registry)
+dataframe_agent = DataFrameAgent(registry)
+visualization_agent = VisualizationAgent(registry)
+transformer_agent = TransformerAgent(registry)
+analysis_agent = AnalysisAgent(registry)
+
+METADATA_CONTEXT_WORDS = [
+    "column", "columns", "numeric", "categorical", "missing", "fields", "field"
+]
+
+
+def _validate_plan_column(plan):
+    """
+    If the LLM plan specifies a column, verify it actually exists in the dataset.
+    Returns (ok, error_message). ok=True means safe to proceed.
+    """
+    column = plan.get("column")
+    dataset = plan.get("dataset")
+
+    if not column or not dataset:
+        return True, None
+
+    try:
+        info = registry.get_info(dataset)
+        columns = [c.lower() for c in info.get("columns", [])]
+        if column.lower() not in columns:
+            msg = (
+                f"Column '{column}' does not exist in dataset '{dataset}'. "
+                f"Available columns: {', '.join(info.get('columns', []))}"
+            )
+            logger.warning(f"Column validation failed | {msg}")
+            return False, msg
+    except Exception as e:
+        logger.error(f"Column validation error | {e}")
+        return False, f"Could not validate column '{column}' in dataset '{dataset}'."
+
+    return True, None
+
+
+def _is_list_with_context(command):
+    """
+    Returns True if 'list' is used in a dataset-specific context
+    (e.g. 'list all columns in leads') rather than a bare 'list datasets' call.
+    """
+    q = command.lower()
+    return any(word in q for word in METADATA_CONTEXT_WORDS)
+
+
+def extract_dataset(command, registry):
+    datasets = registry.list_datasets()
+    words = command.lower().split()
+
+    for word in words:
+        for d in datasets:
+            if word == d.lower():
+                return d
+    return None
+
+
+def handle_command(command):
+
+    try:
+        
+        parts = command.strip().split()
+
+        if not parts:
+            return ""
+
+        action = parts[0].lower()
+
+        # ── LOAD ──────────────────────────────────────────────────────────────
+        if action == "load":
+            if len(parts) < 2:
+                return "Please provide a dataset path."
+
+            path = parts[1]
+            name, df = load_dataset(path)
+            schema = extract_schema(df)
+            registry.register_dataset(name, df, schema)
+            return f"Dataset '{name}' loaded."
+
+        # ── LIST ──────────────────────────────────────────────────────────────
+        # If the user says "list columns in X" or "list numeric in X" etc.,
+        # route to metadata_agent instead of showing all datasets.
+        if action == "list":
+            if _is_list_with_context(command):
+                result = metadata_agent.handle(command)
+                console.print(result)
+                console.print(registry.list_datasets())
+                return ""
+
+            datasets = registry.list_datasets()
+
+            if not datasets:
+                return "No datasets loaded."
+
+            table = Table(title="Datasets")
+            table.add_column("Name")
+
+            for d in datasets:
+                table.add_row(d)
+
+            console.print(table)
+            return ""
+        
+        #── DELETE ──────────────────────────────────────────────────────────────
+        if "delete" in command:
+            dataset = extract_dataset(command, registry)
+
+            if not dataset:
+                return "Please specify dataset to delete (e.g., 'delete leads')"
+
+            return registry.delete_dataset(dataset)
+        # ── INFO ──────────────────────────────────────────────────────────────
+        if action == "info":
+            if len(parts) < 2:
+                return "Provide dataset name."
+
+            name = parts[1]
+            meta = registry.get_info(name)
+
+            rows = meta.get("rows", "unknown")
+            cols = meta.get("columns", [])
+            numeric = meta.get("numeric_columns", [])
+            categorical = meta.get("categorical_columns", [])
+            column_types = meta.get("column_types", {})
+
+            table = Table(title=f"Dataset Info: {name}")
+            table.add_column("Property")
+            table.add_column("Value")
+
+            table.add_row("Rows", str(rows))
+            table.add_row("Columns", str(len(cols)))
+            table.add_row("Numeric Columns", ", ".join(numeric) if numeric else "None")
+            table.add_row("Categorical Columns", ", ".join(categorical) if categorical else "None")
+            table.add_row(
+                "Column Types",
+                ", ".join([f"{k}:{v}" for k, v in column_types.items()])
+            )
+
+            console.print(table)
+            return ""
+
+        # ── DESCRIBE ──────────────────────────────────────────────────────────
+        if action == "describe":
+            if len(parts) < 2:
+                return "Provide dataset name."
+
+            name = parts[1]
+            df = registry.load_dataframe(name)
+            console.print(df.describe().round(2))
+            return ""
+
+        # ── EXIT ──────────────────────────────────────────────────────────────
+        if action == "exit":
+            return "exit"
+        
+        # ── Analyze ──────────────────────────────────────────────────────────────
+        if action in {"analyze", "analyse"}:  
+            return analysis_agent.handle(command)
+        
+        # ── HELP ──────────────────────────────────────────────────────────────
+        if action == "help":
+            table = Table(title="EDA Explorer Commands")
+
+            table.add_column("Command")
+            table.add_column("Description")
+
+            # ---------- DATASET ----------
+            table.add_row("load <file_path>", "Load dataset (auto converts to parquet)")
+            table.add_row("delete <dataset>", "Delete dataset (parquet + metadata)")
+            table.add_row("delete all", "Delete ALL datasets")
+            table.add_row("list", "List available datasets")
+
+            # ---------- METADATA ----------
+            table.add_row("info <dataset>", "Show dataset metadata")
+            table.add_row("columns <dataset>", "Show column names")
+            table.add_row("shape <dataset>", "Show dataset size")
+            table.add_row("list columns in <dataset>", "List columns (metadata agent)")
+
+            # ---------- DATA PREVIEW ----------
+            table.add_row("head <dataset> [n]", "Preview first rows")
+            table.add_row("describe <dataset>", "Statistical summary")
+
+            # ---------- ANALYSIS ----------
+            table.add_row("analyze <dataset>", "Full EDA analysis (quality + warnings)")
+            table.add_row("missing <dataset>", "Show missing values")
+            table.add_row("duplicates <dataset>", "Show duplicate rows")
+            table.add_row("correlation <dataset>", "Correlation matrix")
+
+            # ---------- NATURAL LANGUAGE ----------
+            table.add_row("NL: show top 10 rows in <dataset>", "Row preview")
+            table.add_row("NL: how many rows in <dataset>", "Row count")
+            table.add_row("NL: average <column> in <dataset>", "Column mean")
+            table.add_row("NL: histogram <column> in <dataset>", "Histogram")
+            table.add_row("NL: bar chart <column> in <dataset>", "Bar chart")
+
+            # ---------- SYSTEM ----------
+            table.add_row("exit", "Quit program")
+
+            console.print(table)
+
+        # ── COLUMNS ───────────────────────────────────────────────────────────
+        if action == "columns":
+            if len(parts) < 2:
+                return "Provide dataset name."
+
+            name = parts[1]
+            meta = registry.get_info(name)
+            cols = meta.get("columns", [])
+
+            table = Table(title=f"Columns: {name}")
+            table.add_column("Column Name")
+
+            for col in cols:
+                table.add_row(col)
+
+            console.print(table)
+            return ""
+
+        # ── SHAPE ─────────────────────────────────────────────────────────────
+        if action == "shape":
+            if len(parts) < 2:
+                return "Provide dataset name."
+
+            name = parts[1]
+            meta = registry.get_info(name)
+            rows = meta.get("rows", "unknown")
+            cols = len(meta.get("columns", []))
+
+            console.print(f"\nRows: {rows}")
+            console.print(f"Columns: {cols}\n")
+            return ""
+
+        # ── HEAD ──────────────────────────────────────────────────────────────
+        if action == "head":
+            if len(parts) < 2:
+                return "Provide dataset name."
+
+            name = parts[1]
+            n = 5
+
+            if len(parts) == 3:
+                try:
+                    n = int(parts[2])
+                except Exception:
+                    pass
+
+            df = registry.load_dataframe(name)
+            console.print(df.head(n))
+            return ""
+
+                # ── AGENT ROUTING ─────────────────────────────────────────────────────
+        # LLM planner is tried first; falls back to rule-based router if the
+        # key is missing or the LLM call fails.
+
+        plan = llm_planner.plan(command)
+        agent_name = plan["agent"] if plan else router.route(command)
+
+        # Column validation: if the LLM suggested a column, confirm it exists
+        if plan and plan.get("column"):
+            ok, err = _validate_plan_column(plan)
+            if not ok:
+                return err
+
+        agent_map = {
+            "metadata_agent":      metadata_agent,
+            "dataframe_agent":     dataframe_agent,
+            "visualization_agent": visualization_agent,
+            "transformer_agent":   transformer_agent,
+            "analysis_agent":      analysis_agent,
+        }
+
+        if agent_name in agent_map:
+            agent = agent_map[agent_name]
+
+            # ---- SPECIAL HANDLING ----
+
+            # Transformer agent uses full plan
+            if agent_name == "transformer_agent" and plan:
+                result = agent.handle(command, plan=plan)
+
+            #  Analysis agent gets dataset directly
+            elif agent_name == "analysis_agent":
+                dataset = plan.get("dataset") if plan else None
+
+                # fallback if dataset missing
+                if not dataset:
+                    datasets = registry.list_datasets()
+                    if not datasets:
+                        return "No datasets available."
+                    dataset = datasets[0]
+
+                result = agent.handle(dataset)
+
+            # Default agents
+            else:
+                result = agent.handle(command)
+
+            console.print(result)
+            return ""
+
+        return "Unknown command. Type 'help' to see available commands."
+    except Exception as e:
+        logger.error(f"Command failed: {command} | {e}")
+        return f"Error: {e}"

core/llm_planner.py

@@ -0,0 +1,305 @@
+import json
+import os
+import requests
+from pathlib import Path
+from utils.logger import logger
+
+
+# OLLAMA_BASE_URL = os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434")
+# OLLAMA_MODEL = os.environ.get("OLLAMA_MODEL", "hf.co/bartowski/gemma-2-2b-it-GGUF:Q5_K_M")
+
+
+class LLMPlanner:
+    """
+    Schema-aware query planner using Ollama (default: gemma3).
+
+    Before calling the LLM, the planner reads the dataset's JSON metadata
+    (fast — no parquet touch) and injects column names and types into the
+    prompt so the model can resolve natural-language column references to
+    their exact names and pick the right preprocessing operation.
+
+    Falls back gracefully if Ollama is unavailable.
+    """
+
+    VALID_AGENTS = {
+        "metadata_agent",
+        "dataframe_agent",
+        "visualization_agent",
+        "transformer_agent",
+        "analysis_agent",
+    }
+
+    VALID_OPERATIONS = {
+        # metadata
+        "columns", "numeric_columns", "categorical_columns",
+        "missing_values", "column_count",
+        # dataframe
+        "head", "row_count", "mean", "max", "min",
+        # visualization
+        "histogram", "bar_chart",
+        # transformer — cleaning
+        "drop_duplicates", "fill_nulls", "drop_column",
+        "drop_constant_columns", "strip_whitespace",
+        "drop_missing_rows", "drop_missing_cols",
+        # transformer — explicit fill strategies
+        "fill_mean", "fill_median", "fill_mode", "fill_zero",
+        # transformer — transforms
+        "normalize", "standardize", "encode", "onehot", "rename",
+        # analysis
+        "analyze","analyse"
+    }
+
+    SYSTEM_PROMPT = """\
+You are a planner for a data analysis CLI system.
+
+Convert the user query into a JSON execution plan.
+
+Return ONLY valid JSON with exactly this structure:
+{
+  "agent": "<agent_name>",
+  "operation": "<operation>",
+  "dataset": "<exact dataset name or null>",
+  "column": "<exact column name from schema or null>"
+}
+
+Valid agents:
+  metadata_agent      — schema / structure queries
+  dataframe_agent     — statistics, row previews
+  visualization_agent — charts
+  transformer_agent   — cleaning, filling, encoding, scaling
+
+metadata_agent operations:
+  columns, numeric_columns, categorical_columns, missing_values, column_count
+
+dataframe_agent operations:
+  head, row_count, mean, max, min
+
+visualization_agent operations:
+  histogram, bar_chart
+
+transformer_agent operations:
+  Cleaning  : drop_duplicates, drop_column, drop_constant_columns,
+              strip_whitespace, drop_missing_rows, drop_missing_cols
+  Filling   : fill_nulls (smart — auto picks mean/median/mode),
+              fill_mean, fill_median, fill_mode, fill_zero
+  Scaling   : normalize (min-max [0,1]), standardize (z-score)
+  Encoding  : encode (label), onehot (one-hot / get_dummies)
+  Other     : rename
+
+Rules:
+- Output ONLY JSON — no explanation, no markdown, no extra keys.
+- Use the EXACT column name from the schema context provided.
+- If the query covers all columns (e.g. "fill all nulls"), set column to null.
+- For queries about listing/showing structure → metadata_agent.
+- For queries about previewing data or computing statistics → dataframe_agent.
+- For fill operations: choose fill_mean/fill_median/fill_mode/fill_zero when the
+  user explicitly names a strategy; use fill_nulls when they don't.
+"""
+
+    def __init__(self):
+        self.enabled = True
+        # logger.info(f"LLMPlanner ready | model={OLLAMA_MODEL} | base={OLLAMA_BASE_URL}")
+
+    # ── schema context ─────────────────────────────────────────────────────
+
+    def _load_schema_context(self, query):
+        """
+        Scan the query for a known dataset name and load its JSON metadata.
+        Returns a compact schema string for injection into the LLM prompt.
+        Reads only the tiny JSON file — parquet is never touched.
+        """
+        meta_dir = Path("data/metadata")
+        if not meta_dir.exists():
+            return None, ""
+
+        q = query.lower()
+        for meta_file in sorted(meta_dir.glob("*.json")):
+            name = meta_file.stem
+            if name.endswith("_clean"):
+                continue
+            if name.lower() not in q:
+                continue
+            try:
+                with open(meta_file) as f:
+                    schema = json.load(f)
+
+                cols      = schema.get("columns", [])
+                numeric   = schema.get("numeric_columns", [])
+                cats      = schema.get("categorical_columns", [])
+                col_types = schema.get("column_types", {})
+                rows      = schema.get("rows", "?")
+
+                lines = [
+                    f"Dataset '{name}' ({rows} rows, {len(cols)} columns):",
+                    f"  All columns  : {', '.join(cols)}",
+                    f"  Numeric      : {', '.join(numeric) if numeric else 'none'}",
+                    f"  Categorical  : {', '.join(cats) if cats else 'none'}",
+                    f"  Column types : {', '.join(f'{k}:{v}' for k, v in col_types.items())}",
+                ]
+                return name, "\n".join(lines)
+            except Exception as e:
+                logger.warning(f"Schema load failed for '{name}' | {e}")
+
+        return None, ""
+
+    # ── ollama call ────────────────────────────────────────────────────────
+
+    # def _call_ollama(self, user_query, schema_context=""):
+    #     """POST to local Ollama API and return the raw response string."""
+    #     try:
+    #         schema_block = (
+    #             f"\n\nSchema context (use exact column names from here):\n{schema_context}"
+    #             if schema_context else ""
+    #         )
+    #         prompt = f"{self.SYSTEM_PROMPT}{schema_block}\n\nUser Query: {user_query}\n\nJSON:"
+
+    #         response = requests.post(
+    #             f"{OLLAMA_BASE_URL}/api/generate",
+    #             json={
+    #                 "model": OLLAMA_MODEL,
+    #                 "prompt": prompt,
+    #                 "format": "json",
+    #                 "stream": False,
+    #                 "options": {
+    #                     "temperature": 0,
+    #                     "top_p": 0.9,
+    #                     "num_predict": 100,
+    #                     "stop": ["\n\n"],
+    #                 },
+    #             },
+    #             timeout=30,
+    #         )
+    #         response.raise_for_status()
+    #         return response.json().get("response", "").strip()
+
+    #     except Exception as e:
+    #         logger.error(f"Ollama call failed | {e}")
+    #         return None
+
+
+    # ── ollama call with huggingface fallback ────────────────────────────────
+
+    def _call_ollama(self, user_query, schema_context=""):
+        """
+        POST to local Ollama API. Falls back to Hugging Face Serverless API 
+        if local service is unavailable, missing, or times out.
+        """
+        # Add this environment check at the top of your function
+        if os.environ.get("ENVIRONMENT") == "production":
+            logger.info("Production mode: Skipping local Ollama check. Routing directly to Hugging Face...")
+            # Jump straight to your Hugging Face API request logic here!
+        schema_block = (
+            f"\n\nSchema context (use exact column names from here):\n{schema_context}"
+            if schema_context else ""
+        )
+        prompt = f"{self.SYSTEM_PROMPT}{schema_block}\n\nUser Query: {user_query}\n\nJSON:"
+
+        # 1. Try Local Ollama First
+        try:
+            logger.info("Attempting local Ollama generation...")
+            # response = requests.post(
+            #     f"{OLLAMA_BASE_URL}/api/generate",
+            #     json={
+            #         "model": OLLAMA_MODEL,
+            #         "prompt": prompt,
+            #         "format": "json",
+            #         "stream": False,
+            #         "options": {
+            #             "temperature": 0,
+            #             "top_p": 0.9,
+            #             "num_predict": 100,
+            #             "stop": ["\n\n"],
+            #         },
+            #     },
+            #     timeout=5, # Reduced timeout so fallback triggers rapidly if offline
+            # )
+            # response.raise_for_status()
+            # return response.json().get("response", "").strip()
+
+        except Exception as local_err:
+            logger.warning(f"Local Ollama unavailable ({local_err}). Routing fallback to Hugging Face...")
+            
+            # 2. Hugging Face Serverless Fallback
+            hf_token = os.environ.get("HF_TOKEN")
+            if not hf_token:
+                logger.error("Hugging Face fallback skipped: HF_TOKEN environment variable not set.")
+                return None
+                
+            try:
+                # We use HF's serverless OpenAI-compatible Router endpoint
+                hf_url = "https://router.huggingface.co/v1/chat/completions"
+                
+                headers = {
+                    "Authorization": f"Bearer {hf_token}",
+                    "Content-Type": "application/json"
+                }
+                
+                # Format to structure for standard OpenAI/HF chat endpoint specs
+                hf_payload = {
+                    "model": "google/gemma-3-12b-it", # Fallback leverages beefier cloud model
+                    "messages": [
+                        {"role": "user", "content": prompt}
+                    ],
+                    "temperature": 0,
+                    "max_tokens": 100,
+                    "response_format": {"type": "json_object"} # Forces strict JSON format out of HF
+                }
+                
+                hf_response = requests.post(hf_url, headers=headers, json=hf_payload, timeout=15)
+                hf_response.raise_for_status()
+                
+                # Extract text out of OpenAI spec completion block
+                result_json = hf_response.json()
+                content = result_json['choices'][0]['message']['content'].strip()
+                
+                logger.info("Successfully fetched plan from Hugging Face.")
+                return content
+                
+            except Exception as hf_err:
+                logger.error(f"Hugging Face fallback also failed | {hf_err}")
+                return None
+
+    # ── public API ─────────────────────────────────────────────────────────
+
+    def plan(self, query):
+        """
+        Return a validated execution plan dict, or None if unavailable.
+        The plan always contains: agent, operation, dataset, column.
+        """
+        _, schema_ctx = self._load_schema_context(query)
+        content = self._call_ollama(query, schema_ctx)
+
+        if not content:
+            return None
+
+        try:
+            if content.startswith("```"):
+                content = content.split("```")[1]
+                if content.startswith("json"):
+                    content = content[4:]
+
+            plan = json.loads(content)
+
+            agent     = plan.get("agent")
+            operation = plan.get("operation")
+
+            if agent not in self.VALID_AGENTS:
+                logger.error(f"LLM returned invalid agent: {agent!r}")
+                return None
+
+            if operation not in self.VALID_OPERATIONS:
+                logger.error(f"LLM returned invalid operation: {operation!r}")
+                return None
+
+            plan.setdefault("dataset", None)
+            plan.setdefault("column", None)
+
+            logger.info(f"LLMPlanner plan → {plan}")
+            return plan
+
+        except json.JSONDecodeError as e:
+            logger.error(f"LLM response not valid JSON | {e} | raw: {content!r}")
+            return None
+        except Exception as e:
+            logger.error(f"LLMPlanner error | {e}")
+            return None

core/query_router.py

@@ -0,0 +1,96 @@
+from utils.logger import logger
+
+
+class QueryRouter:
+    """
+    Rule-based fallback router.
+    Order matters: transformer action words are checked first so that queries
+    like 'drop column X' or 'impute missing' don't get swallowed by the
+    metadata keyword list.
+    """
+
+    # Transformer keywords take top priority — they are explicit action verbs.
+    TRANSFORMER_KEYWORDS = [
+        "normalize",
+        "standardize",
+        "zscore",
+        "z-score",
+        "scale",
+        "encode",
+        "onehot",
+        "one-hot",
+        "one hot",
+        "dummies",
+        "drop",
+        "fill",
+        "impute",
+        "rename",
+        "strip",
+        "duplicate",
+        "constant",
+        "whitespace",
+        "dropna",
+    ]
+
+    # Metadata keywords — structural / schema queries.
+    METADATA_KEYWORDS = [
+        "column",
+        "numeric",
+        "categorical",
+        "missing",
+        "schema",
+        "fields",
+        "field",
+    ]
+
+    # DataFrame / statistics keywords.
+    DATAFRAME_KEYWORDS = [
+        "average",
+        "mean",
+        "median",
+        "max",
+        "min",
+        "top",
+        "count",
+        "rows",
+        "sum",
+        "highest",
+        "lowest",
+    ]
+
+    # Visualisation keywords.
+    VISUAL_KEYWORDS = [
+        "plot",
+        "graph",
+        "scatter",
+        "hist",
+        "bar",
+        "chart",
+        "histogram",
+        "distribution",
+    ]
+
+    def route(self, query):
+        q = query.lower()
+
+        if any(word in q for word in self.TRANSFORMER_KEYWORDS):
+            logger.info("Routing → transformer_agent")
+            return "transformer_agent"
+
+        if any(word in q for word in self.METADATA_KEYWORDS):
+            logger.info("Routing → metadata_agent")
+            return "metadata_agent"
+
+        if any(word in q for word in self.DATAFRAME_KEYWORDS):
+            logger.info("Routing → dataframe_agent")
+            return "dataframe_agent"
+
+        if any(word in q for word in self.VISUAL_KEYWORDS):
+            logger.info("Routing → visualization_agent")
+            return "visualization_agent"
+        
+        if "analyze" in query or "analysis" in query or "analyse" in query:
+            return "analysis_agent"
+
+        logger.warning(f"No route matched for query: {query}")
+        return "unknown command"

data/dataframe_store.py

@@ -0,0 +1,58 @@
+class DataFrameStore:
+    """
+    A simple in-memory manager for storing and accessing multiple datasets.
+
+    Each dataset is stored with:
+    - original: the untouched DataFrame
+    - working: a copy used for transformations
+    - schema: metadata describing the dataset structure
+    """
+
+    def __init__(self):
+        """Initialize an empty dataset store."""
+        self.datasets = {}
+
+    def add_dataset(self, name, df, schema):
+        """
+        Add a dataset to the store.
+
+        Parameters
+        ----------
+        name : str
+            Unique name used to identify the dataset.
+        df : pandas.DataFrame
+            The DataFrame to store.
+        schema : dict
+            Metadata describing column types or structure.
+        """
+        if name in self.datasets:
+            raise ValueError(f"Dataset '{name}' already loaded")
+
+        self.datasets[name] = {
+            "original": df,
+            "working": df.copy(),
+            "schema": schema
+        }
+
+    def list_datasets(self):
+        """
+        Return a list of all dataset names currently stored.
+        """
+        return list(self.datasets.keys())
+
+    def get_dataset(self, name):
+        """
+        Retrieve the dataset dictionary for a given dataset name.
+
+        Returns
+        -------
+        dict
+            Contains 'original', 'working', and 'schema'.
+        """
+        return self.datasets.get(name)
+
+    def get_schema(self, name):
+        """
+        Get the schema metadata for a specific dataset.
+        """
+        return self.datasets[name]["schema"]

data/loader.py

@@ -0,0 +1,34 @@
+import pandas as pd
+import os
+from pathlib import Path
+
+def load_dataset(path):
+    #strips path with quotes. 
+    path = path.strip().strip('"').strip("'")
+    if not os.path.exists(path):
+        raise FileNotFoundError("Dataset file not found")
+    
+    
+    name = Path(path).stem
+
+    if path.endswith(".csv"):
+
+        encodings = ["utf-8", "latin1", "cp1252"]
+
+        for enc in encodings:
+            try:
+                df = pd.read_csv(path, encoding=enc)
+                print(f"Loaded CSV using encoding: {enc}")
+                return name, df
+            except UnicodeDecodeError:
+                continue
+
+        raise ValueError("Could not decode CSV file with common encodings.")
+
+    elif path.endswith(".xlsx"):
+
+        df = pd.read_excel(path)
+        return name, df
+
+    else:
+        raise ValueError("Only CSV and XLSX supported")

data/metadata/Customer_Churn.json

@@ -0,0 +1,97 @@
+{
+  "rows": 7043,
+  "columns": [
+    "customerID",
+    "gender",
+    "SeniorCitizen",
+    "Partner",
+    "Dependents",
+    "tenure",
+    "PhoneService",
+    "MultipleLines",
+    "InternetService",
+    "OnlineSecurity",
+    "OnlineBackup",
+    "DeviceProtection",
+    "TechSupport",
+    "StreamingTV",
+    "StreamingMovies",
+    "Contract",
+    "PaperlessBilling",
+    "PaymentMethod",
+    "MonthlyCharges",
+    "TotalCharges",
+    "Churn"
+  ],
+  "numeric_columns": [
+    "SeniorCitizen",
+    "tenure",
+    "MonthlyCharges"
+  ],
+  "categorical_columns": [
+    "customerID",
+    "gender",
+    "Partner",
+    "Dependents",
+    "PhoneService",
+    "MultipleLines",
+    "InternetService",
+    "OnlineSecurity",
+    "OnlineBackup",
+    "DeviceProtection",
+    "TechSupport",
+    "StreamingTV",
+    "StreamingMovies",
+    "Contract",
+    "PaperlessBilling",
+    "PaymentMethod",
+    "TotalCharges",
+    "Churn"
+  ],
+  "missing_values": {
+    "customerID": 0.0,
+    "gender": 0.0,
+    "SeniorCitizen": 0.0,
+    "Partner": 0.0,
+    "Dependents": 0.0,
+    "tenure": 0.0,
+    "PhoneService": 0.0,
+    "MultipleLines": 0.0,
+    "InternetService": 0.0,
+    "OnlineSecurity": 0.0,
+    "OnlineBackup": 0.0,
+    "DeviceProtection": 0.0,
+    "TechSupport": 0.0,
+    "StreamingTV": 0.0,
+    "StreamingMovies": 0.0,
+    "Contract": 0.0,
+    "PaperlessBilling": 0.0,
+    "PaymentMethod": 0.0,
+    "MonthlyCharges": 0.0,
+    "TotalCharges": 0.0,
+    "Churn": 0.0
+  },
+  "column_types": {
+    "customerID": "str",
+    "gender": "str",
+    "SeniorCitizen": "int64",
+    "Partner": "str",
+    "Dependents": "str",
+    "tenure": "int64",
+    "PhoneService": "str",
+    "MultipleLines": "str",
+    "InternetService": "str",
+    "OnlineSecurity": "str",
+    "OnlineBackup": "str",
+    "DeviceProtection": "str",
+    "TechSupport": "str",
+    "StreamingTV": "str",
+    "StreamingMovies": "str",
+    "Contract": "str",
+    "PaperlessBilling": "str",
+    "PaymentMethod": "str",
+    "MonthlyCharges": "float64",
+    "TotalCharges": "str",
+    "Churn": "str"
+  }
+}

data/metadata/Loan_default.json

@@ -0,0 +1,85 @@
+{
+  "rows": 255347,
+  "columns": [
+    "LoanID",
+    "Age",
+    "Income",
+    "LoanAmount",
+    "CreditScore",
+    "MonthsEmployed",
+    "NumCreditLines",
+    "InterestRate",
+    "LoanTerm",
+    "DTIRatio",
+    "Education",
+    "EmploymentType",
+    "MaritalStatus",
+    "HasMortgage",
+    "HasDependents",
+    "LoanPurpose",
+    "HasCoSigner",
+    "Default"
+  ],
+  "numeric_columns": [
+    "Age",
+    "Income",
+    "LoanAmount",
+    "CreditScore",
+    "MonthsEmployed",
+    "NumCreditLines",
+    "InterestRate",
+    "LoanTerm",
+    "DTIRatio",
+    "Default"
+  ],
+  "categorical_columns": [
+    "LoanID",
+    "Education",
+    "EmploymentType",
+    "MaritalStatus",
+    "HasMortgage",
+    "HasDependents",
+    "LoanPurpose",
+    "HasCoSigner"
+  ],
+  "missing_values": {
+    "LoanID": 0.0,
+    "Age": 0.0,
+    "Income": 0.0,
+    "LoanAmount": 0.0,
+    "CreditScore": 0.0,
+    "MonthsEmployed": 0.0,
+    "NumCreditLines": 0.0,
+    "InterestRate": 0.0,
+    "LoanTerm": 0.0,
+    "DTIRatio": 0.0,
+    "Education": 0.0,
+    "EmploymentType": 0.0,
+    "MaritalStatus": 0.0,
+    "HasMortgage": 0.0,
+    "HasDependents": 0.0,
+    "LoanPurpose": 0.0,
+    "HasCoSigner": 0.0,
+    "Default": 0.0
+  },
+  "column_types": {
+    "LoanID": "str",
+    "Age": "int64",
+    "Income": "int64",
+    "LoanAmount": "int64",
+    "CreditScore": "int64",
+    "MonthsEmployed": "int64",
+    "NumCreditLines": "int64",
+    "InterestRate": "float64",
+    "LoanTerm": "int64",
+    "DTIRatio": "float64",
+    "Education": "str",
+    "EmploymentType": "str",
+    "MaritalStatus": "str",
+    "HasMortgage": "str",
+    "HasDependents": "str",
+    "LoanPurpose": "str",
+    "HasCoSigner": "str",
+    "Default": "int64"
+  }
+}

data/metadata/creditcard.json

@@ -0,0 +1,136 @@
+{
+  "rows": 284807,
+  "columns": [
+    "Time",
+    "V1",
+    "V2",
+    "V3",
+    "V4",
+    "V5",
+    "V6",
+    "V7",
+    "V8",
+    "V9",
+    "V10",
+    "V11",
+    "V12",
+    "V13",
+    "V14",
+    "V15",
+    "V16",
+    "V17",
+    "V18",
+    "V19",
+    "V20",
+    "V21",
+    "V22",
+    "V23",
+    "V24",
+    "V25",
+    "V26",
+    "V27",
+    "V28",
+    "Amount",
+    "Class"
+  ],
+  "numeric_columns": [
+    "Time",
+    "V1",
+    "V2",
+    "V3",
+    "V4",
+    "V5",
+    "V6",
+    "V7",
+    "V8",
+    "V9",
+    "V10",
+    "V11",
+    "V12",
+    "V13",
+    "V14",
+    "V15",
+    "V16",
+    "V17",
+    "V18",
+    "V19",
+    "V20",
+    "V21",
+    "V22",
+    "V23",
+    "V24",
+    "V25",
+    "V26",
+    "V27",
+    "V28",
+    "Amount",
+    "Class"
+  ],
+  "categorical_columns": [],
+  "missing_values": {
+    "Time": 0.0,
+    "V1": 0.0,
+    "V2": 0.0,
+    "V3": 0.0,
+    "V4": 0.0,
+    "V5": 0.0,
+    "V6": 0.0,
+    "V7": 0.0,
+    "V8": 0.0,
+    "V9": 0.0,
+    "V10": 0.0,
+    "V11": 0.0,
+    "V12": 0.0,
+    "V13": 0.0,
+    "V14": 0.0,
+    "V15": 0.0,
+    "V16": 0.0,
+    "V17": 0.0,
+    "V18": 0.0,
+    "V19": 0.0,
+    "V20": 0.0,
+    "V21": 0.0,
+    "V22": 0.0,
+    "V23": 0.0,
+    "V24": 0.0,
+    "V25": 0.0,
+    "V26": 0.0,
+    "V27": 0.0,
+    "V28": 0.0,
+    "Amount": 0.0,
+    "Class": 0.0
+  },
+  "column_types": {
+    "Time": "float64",
+    "V1": "float64",
+    "V2": "float64",
+    "V3": "float64",
+    "V4": "float64",
+    "V5": "float64",
+    "V6": "float64",
+    "V7": "float64",
+    "V8": "float64",
+    "V9": "float64",
+    "V10": "float64",
+    "V11": "float64",
+    "V12": "float64",
+    "V13": "float64",
+    "V14": "float64",
+    "V15": "float64",
+    "V16": "float64",
+    "V17": "float64",
+    "V18": "float64",
+    "V19": "float64",
+    "V20": "float64",
+    "V21": "float64",
+    "V22": "float64",
+    "V23": "float64",
+    "V24": "float64",
+    "V25": "float64",
+    "V26": "float64",
+    "V27": "float64",
+    "V28": "float64",
+    "Amount": "float64",
+    "Class": "int64"
+  }
+}

data/metadata/titanic.json

@@ -0,0 +1,61 @@
+{
+  "rows": 891,
+  "columns": [
+    "PassengerId",
+    "Survived",
+    "Pclass",
+    "Name",
+    "Sex",
+    "Age",
+    "SibSp",
+    "Parch",
+    "Ticket",
+    "Fare",
+    "Cabin",
+    "Embarked"
+  ],
+  "numeric_columns": [
+    "PassengerId",
+    "Survived",
+    "Pclass",
+    "Age",
+    "SibSp",
+    "Parch",
+    "Fare"
+  ],
+  "categorical_columns": [
+    "Name",
+    "Sex",
+    "Ticket",
+    "Cabin",
+    "Embarked"
+  ],
+  "missing_values": {
+    "PassengerId": 0.0,
+    "Survived": 0.0,
+    "Pclass": 0.0,
+    "Name": 0.0,
+    "Sex": 0.0,
+    "Age": 0.1987,
+    "SibSp": 0.0,
+    "Parch": 0.0,
+    "Ticket": 0.0,
+    "Fare": 0.0,
+    "Cabin": 0.771,
+    "Embarked": 0.0022
+  },
+  "column_types": {
+    "PassengerId": "int64",
+    "Survived": "int64",
+    "Pclass": "int64",
+    "Name": "str",
+    "Sex": "str",
+    "Age": "float64",
+    "SibSp": "int64",
+    "Parch": "int64",
+    "Ticket": "str",
+    "Fare": "float64",
+    "Cabin": "str",
+    "Embarked": "str"
+  }
+}

data/registry.py

@@ -0,0 +1,164 @@
+import json
+from pathlib import Path
+import pandas as pd
+from utils.logger import logger
+import os 
+DATASET_DIR = Path("data/datasets")
+METADATA_DIR = Path("data/metadata")
+
+DATASET_DIR.mkdir(parents=True, exist_ok=True)
+METADATA_DIR.mkdir(parents=True, exist_ok=True)
+
+
+class DatasetRegistry:
+
+    def __init__(self):
+
+        self.datasets = {}
+
+        self._load_existing()
+
+    def _load_existing(self):
+
+        try:
+
+            for meta_file in METADATA_DIR.glob("*.json"):
+
+                name = meta_file.stem
+
+                with open(meta_file, "r") as f:
+                    metadata = json.load(f)
+
+                self.datasets[name] = metadata
+                logger.info(f"Loaded {len(self.datasets)} datasets into registry")
+
+        except Exception as e:
+            logger.error(f"Registry loading failed | {e}")
+
+    def delete_dataset(self, name):
+        try:
+            deleted_files = []
+
+            dataset_dir = "data/datasets"
+            metadata_dir = "data/metadata"
+
+            # Possible variations
+            variants = [name, f"{name}_clean"]
+
+            for variant in variants:
+                parquet_path = os.path.join(dataset_dir, f"{variant}.parquet")
+                metadata_path = os.path.join(metadata_dir, f"{variant}.json")
+
+                if os.path.exists(parquet_path):
+                    os.remove(parquet_path)
+                    deleted_files.append(parquet_path)
+
+                if os.path.exists(metadata_path):
+                    os.remove(metadata_path)
+                    deleted_files.append(metadata_path)
+
+            if not deleted_files:
+                return f"No dataset found for '{name}'"
+
+            logger.info(f"Deleted dataset {name} | Files: {deleted_files}")
+
+            return f"Deleted dataset '{name}' successfully."
+
+        except Exception as e:
+            logger.error(f"Delete failed | {e}")
+            return f"Failed to delete dataset '{name}'"
+
+    def register_dataset(self, name, df, schema):
+
+        try:
+
+            if name in self.datasets:
+                raise ValueError(f"Dataset '{name}' already exists")
+
+            parquet_path = DATASET_DIR / f"{name}.parquet"
+            meta_path = METADATA_DIR / f"{name}.json"
+
+            df.to_parquet(parquet_path)
+
+            with open(meta_path, "w") as f:
+                json.dump(schema, f, indent=2)
+
+            self.datasets[name] = schema
+
+            logger.info(f"Dataset registered | {name}")
+
+        except Exception as e:
+            logger.error(f"Dataset registration failed | {e}")
+            raise
+
+    def dataset_exists(self, name):
+
+        return name in self.datasets
+
+    def list_datasets(self):
+
+        return list(self.datasets.keys())
+
+    def get_info(self, name):
+
+        if name not in self.datasets:
+            raise ValueError("Dataset not found")
+
+        return self.datasets[name]
+
+    def update_dataset(self, name, df, schema):
+
+        try:
+
+            parquet_path = DATASET_DIR / f"{name}.parquet"
+            meta_path = METADATA_DIR / f"{name}.json"
+
+            df.to_parquet(parquet_path)
+
+            with open(meta_path, "w") as f:
+                json.dump(schema, f, indent=2)
+
+            self.datasets[name] = schema
+
+            logger.info(f"Dataset updated | {name}")
+
+        except Exception as e:
+            logger.error(f"Dataset update failed | {e}")
+            raise
+
+    def load_dataframe(self, name, sample=True, sample_size=50000):
+        try:
+            # ---------- VALIDATION ----------
+            if name not in self.datasets:
+                logger.error(f"Dataset '{name}' not found in registry")
+                raise ValueError(f"Dataset '{name}' not found")
+
+            path = DATASET_DIR / f"{name}.parquet"
+
+            if not path.exists():
+                logger.error(f"Parquet file missing: {path}")
+                raise FileNotFoundError(f"{path} not found")
+
+            logger.info(f"Loading dataset: {name}")
+
+            # ---------- LOAD ----------
+            df = pd.read_parquet(path)
+
+            logger.info(f"Loaded dataset '{name}' | shape={df.shape}")
+
+            # ---------- SMART SAMPLING ----------
+            if sample and len(df) > sample_size:
+                logger.info(
+                    f"Dataset '{name}' is large ({len(df)} rows). "
+                    f"Sampling {sample_size} rows for analysis."
+                )
+
+                df = df.sample(sample_size, random_state=42)
+
+                logger.info(f"Sampled dataset '{name}' | new_shape={df.shape}")
+
+            return df
+
+        except Exception as e:
+            logger.error(f"Failed to load dataset '{name}' | {e}")
+            raise

data/schema_extractor.py

@@ -0,0 +1,29 @@
+import pandas as pd
+
+
+def extract_schema(df: pd.DataFrame):
+
+    schema = {}
+
+    schema["rows"] = len(df)
+
+    schema["columns"] = list(df.columns)
+
+    schema["numeric_columns"] = list(
+        df.select_dtypes(include=["number"]).columns
+    )
+
+    schema["categorical_columns"] = list(
+        df.select_dtypes(include=["object", "category"]).columns
+    )
+
+    schema["missing_values"] = (
+        df.isnull().mean().round(4).to_dict()
+    )
+
+    # NEW FIELD
+    schema["column_types"] = {
+        col: str(dtype) for col, dtype in df.dtypes.items()
+    }
+
+    return schema

instructions/analyze.txt

@@ -0,0 +1,82 @@
+Instruction Set for Generic Dataset Analysis (Read-Only)
+
+1. "Loading dataset '{dataset_name}'..."
+   - Load dataset into memory
+   - Record total rows and columns
+   - Log: "Dataset loaded: {num_rows} rows, {num_columns} columns"
+
+2. "Computing data quality metrics..."
+   - Count total missing values per column and total in dataset
+   - Count total duplicate rows
+   - Log: "Total missing values: {total_missing}, Duplicate rows: {num_duplicates}"
+
+3. "Profiling feature types..."
+   - Identify numeric columns
+   - Identify categorical columns
+   - Compute unique values per categorical column
+   - Log: "Numeric columns: {numeric_columns}, Categorical columns: {categorical_columns}"
+
+4. "Checking basic statistics for numeric features..."
+   - Compute min, max, mean, std for each numeric column
+   - Log: "Numeric summary computed for {column}"
+
+5. "Checking categorical value distributions..."
+   - Compute top 5 categories and their counts for each categorical column
+   - Log: "Categorical summary computed for {column}"
+
+6. "Computing correlation matrix (numeric features only)..."
+   - Compute Pearson correlations between numeric columns
+   - Log: "Correlation matrix computed"
+
+7. "Preparing visualizations in terminal..."
+   - Generate histograms for numeric columns using plotext
+       - Log: "Histogram ready for {column}"
+   - Generate bar charts for categorical columns with cardinality <= 20
+       - Log: "Bar chart ready for {column}"
+   - High-cardinality columns are noted but skipped from plotting
+
+8. "Generating dataset summary (LLM)..."
+   - Summarize:
+       - Rows and columns
+       - Missing value overview
+       - Duplicates overview
+       - Key numeric/categorical insights
+       - Correlation highlights
+   - Log: "Summary generated"
+
+9. "Finalizing terminal dashboard..."
+   - Print structured dashboard like:
+
+      Dataset Analysis: {dataset_name}
+      --------------------------------
+      Rows: {num_rows}
+      Columns: {num_columns}
+
+      Data Quality
+      ------------
+      Total Missing Values : {total_missing}
+      Duplicate Rows       : {num_duplicates}
+
+      Missing by Column
+      -----------------
+      {column1} : {missing_pct1}%
+      {column2} : {missing_pct2}%
+
+      Feature Summary
+      ---------------
+      Numeric Columns      : {numeric_columns}
+      Categorical Columns  : {categorical_columns}
+
+      Correlations (Top pairs)
+      ------------------------
+      {col1} & {col2} : r={value}
+      {col3} & {col4} : r={value}
+
+      Visualizations Ready
+      -------------------
+      [Histograms: {numeric_columns}]
+      [Bar Charts: {categorical_columns <= 20}]
+      
+      LLM Summary
+      -----------
+      {text_summary}

testing/test_agent_routing.py

@@ -0,0 +1,211 @@
+"""
+Agent Routing Test Suite
+========================
+Tests that every query type routes to the correct agent (rule-based router),
+that the 'list' command is correctly disambiguated, and that the LLM plan
+column-validation guard works.
+
+No Ollama required — all tests use the rule-based fallback router directly.
+"""
+
+import sys
+import os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from unittest.mock import patch
+from core.query_router import QueryRouter
+from data.registry import DatasetRegistry
+from cli_app.command_handler import _validate_plan_column, _is_list_with_context
+
+router = QueryRouter()
+
+passed = 0
+failed = 0
+
+
+def run_test(label, got, expected):
+    global passed, failed
+    ok = got == expected
+    tag = "[PASS]" if ok else "[FAIL]"
+    print(f"{tag} {label}")
+    print(f"       Expected : {expected}")
+    print(f"       Got      : {got}\n")
+    if ok:
+        passed += 1
+    else:
+        failed += 1
+
+
+def run_bool_test(label, got, expected=True):
+    global passed, failed
+    ok = bool(got) == expected
+    tag = "[PASS]" if ok else "[FAIL]"
+    print(f"{tag} {label}")
+    print(f"       Expected : {expected}")
+    print(f"       Got      : {got}\n")
+    if ok:
+        passed += 1
+    else:
+        failed += 1
+
+
+print("=" * 60)
+print("  Agent Routing Test Suite")
+print("=" * 60)
+
+
+# ── METADATA AGENT ────────────────────────────────────────────
+print("\n--- Metadata Agent Routing ---\n")
+
+run_test("Columns query",
+         router.route("show all columns in leads"), "metadata_agent")
+run_test("Numeric columns query",
+         router.route("what are the numeric columns in leads"), "metadata_agent")
+run_test("Categorical columns query",
+         router.route("list categorical columns in organizations"), "metadata_agent")
+run_test("Missing values query",
+         router.route("how many missing values in people"), "metadata_agent")
+run_test("Schema query",
+         router.route("show schema for organizations"), "metadata_agent")
+
+
+# ── DATAFRAME AGENT ───────────────────────────────────────────
+print("--- DataFrame Agent Routing ---\n")
+
+run_test("Average query",
+         router.route("average annual_revenue in leads"), "dataframe_agent")
+run_test("Mean query",
+         router.route("mean of employees in organizations"), "dataframe_agent")
+run_test("Max query",
+         router.route("max annual_revenue in leads"), "dataframe_agent")
+run_test("Min query",
+         router.route("min employees in organizations"), "dataframe_agent")
+run_test("Top rows query",
+         router.route("show top 10 rows in leads"), "dataframe_agent")
+run_test("Row count query",
+         router.route("how many rows in leads"), "dataframe_agent")
+
+
+# ── VISUALIZATION AGENT ───────────────────────────────────────
+print("--- Visualization Agent Routing ---\n")
+
+run_test("Histogram query",
+         router.route("histogram of annual_revenue in leads"), "visualization_agent")
+run_test("Bar chart query",
+         router.route("bar chart of industry in leads"), "visualization_agent")
+run_test("Plot query",
+         router.route("plot distribution in organizations"), "visualization_agent")
+run_test("Graph query",
+         router.route("graph of employees"), "visualization_agent")
+
+
+# ── TRANSFORMER AGENT — existing ops ─────────────────────────
+print("--- Transformer Agent Routing (existing ops) ---\n")
+
+run_test("Drop duplicates",
+         router.route("drop duplicates in leads"), "transformer_agent")
+run_test("Fill nulls",
+         router.route("fill nulls in organizations"), "transformer_agent")
+run_test("Normalize",
+         router.route("normalize annual_revenue in leads"), "transformer_agent")
+run_test("Encode",
+         router.route("encode industry in leads"), "transformer_agent")
+run_test("Rename",
+         router.route("rename industry to sector in leads"), "transformer_agent")
+run_test("Drop column (no metadata collision)",
+         router.route("drop column description in leads"), "transformer_agent")
+run_test("Impute (no metadata collision)",
+         router.route("impute missing in organizations"), "transformer_agent")
+run_test("Strip whitespace",
+         router.route("strip whitespace in people"), "transformer_agent")
+
+
+# ── TRANSFORMER AGENT — new preprocessing ops ─────────────────
+print("--- Transformer Agent Routing (new preprocessing ops) ---\n")
+
+run_test("Standardize",
+         router.route("standardize number of employees in organizations"), "transformer_agent")
+run_test("Z-score keyword",
+         router.route("z-score normalize founded in organizations"), "transformer_agent")
+run_test("Zscore keyword",
+         router.route("zscore the index column in leads"), "transformer_agent")
+run_test("One-hot encoding",
+         router.route("one hot encode industry in organizations"), "transformer_agent")
+run_test("Onehot keyword",
+         router.route("onehot encode sex in people"), "transformer_agent")
+run_test("Dummies keyword",
+         router.route("get dummies for industry in organizations"), "transformer_agent")
+run_test("Fill with mean",
+         router.route("fill with mean in organizations"), "transformer_agent")
+run_test("Fill with median",
+         router.route("fill nulls with median in leads"), "transformer_agent")
+run_test("Fill with mode",
+         router.route("fill missing using mode in people"), "transformer_agent")
+run_test("Fill zero",
+         router.route("fill with zero in leads"), "transformer_agent")
+run_test("Drop missing rows",
+         router.route("drop missing rows in organizations"), "transformer_agent")
+run_test("Drop missing cols",
+         router.route("drop missing columns in leads"), "transformer_agent")
+run_test("Dropna keyword",
+         router.route("dropna in organizations"), "transformer_agent")
+
+
+# ── LIST DISAMBIGUATION ───────────────────────────────────────
+print("--- List Ambiguity Detection ---\n")
+
+run_bool_test("'list columns in leads' → metadata context",
+              _is_list_with_context("list columns in leads"), expected=True)
+run_bool_test("'list all numeric columns in people' → metadata context",
+              _is_list_with_context("list all numeric columns in people"), expected=True)
+run_bool_test("'list' alone → no context (dataset list)",
+              _is_list_with_context("list"), expected=False)
+run_bool_test("'list datasets' → no context (dataset list)",
+              _is_list_with_context("list datasets"), expected=False)
+
+
+# ── COLUMN VALIDATION ─────────────────────────────────────────
+print("--- Column Validation (LLM plan guard) ---\n")
+
+registry = DatasetRegistry()
+datasets = registry.list_datasets()
+
+if datasets:
+    sample_dataset = [d for d in datasets if not d.endswith("_clean")][0]
+    info = registry.get_info(sample_dataset)
+    real_columns = info.get("columns", [])
+
+    if real_columns:
+        real_col = real_columns[0]
+
+        with patch("cli_app.command_handler.registry", registry):
+            ok, _ = _validate_plan_column({
+                "agent": "transformer_agent", "operation": "fill_mean",
+                "dataset": sample_dataset, "column": real_col
+            })
+            run_bool_test(f"Valid column '{real_col}' in '{sample_dataset}' → passes",
+                          ok, expected=True)
+
+            ok, _ = _validate_plan_column({
+                "agent": "transformer_agent", "operation": "standardize",
+                "dataset": sample_dataset, "column": "ghost_col_xyz"
+            })
+            run_bool_test("Non-existent column 'ghost_col_xyz' → fails validation",
+                          not ok, expected=True)
+
+            ok, _ = _validate_plan_column({
+                "agent": "transformer_agent", "operation": "drop_missing_rows",
+                "dataset": sample_dataset, "column": None
+            })
+            run_bool_test("Plan with column=None → always passes",
+                          ok, expected=True)
+else:
+    print("[SKIP] No datasets loaded — skipping column validation tests\n")
+
+
+# ── SUMMARY ───────────────────────────────────────────────────
+print("=" * 60)
+print(f"Results: {passed} passed, {failed} failed")
+if failed == 0:
+    print("All tests passed.")
+print("=" * 60)

testing/test_transformer.py

@@ -0,0 +1,369 @@
+import shutil
+import os
+import pandas as pd
+from data.registry import DatasetRegistry
+from data.schema_extractor import extract_schema
+from agents.transformer_agent import TransformerAgent
+
+DATASETS_DIR = "data/datasets"
+METADATA_DIR = "data/metadata"
+DATASETS_BACKUP = "data/datasets_backup"
+METADATA_BACKUP = "data/metadata_backup"
+
+passed = 0
+failed = 0
+
+
+def backup():
+    shutil.copytree(DATASETS_DIR, DATASETS_BACKUP, dirs_exist_ok=True)
+    shutil.copytree(METADATA_DIR, METADATA_BACKUP, dirs_exist_ok=True)
+
+
+def restore():
+    shutil.rmtree(DATASETS_DIR)
+    shutil.rmtree(METADATA_DIR)
+    shutil.copytree(DATASETS_BACKUP, DATASETS_DIR)
+    shutil.copytree(METADATA_BACKUP, METADATA_DIR)
+    shutil.rmtree(DATASETS_BACKUP, ignore_errors=True)
+    shutil.rmtree(METADATA_BACKUP, ignore_errors=True)
+
+
+def fresh():
+    return DatasetRegistry(), None
+
+
+def fresh_agent():
+    registry = DatasetRegistry()
+    return registry, TransformerAgent(registry)
+
+
+def run_test(label, query, check_fn, agent):
+    global passed, failed
+    result = agent.handle(query)
+    try:
+        ok = check_fn(result, agent)
+    except Exception as e:
+        ok = False
+        print(f"  [check error] {e}")
+    tag = "[PASS]" if ok else "[FAIL]"
+    print(f"{tag} {label}")
+    print(f"       Query  : {query}")
+    print(f"       Result : {result}\n")
+    if ok:
+        passed += 1
+    else:
+        failed += 1
+
+
+print("=" * 60)
+print("  Transformer Agent Test Suite")
+print("=" * 60)
+
+backup()
+
+try:
+
+    # ── SAFETY: ORIGINAL IS NEVER MODIFIED ─────────────────
+    print("--- Safety: original dataset is never modified ---\n")
+
+    registry, agent = fresh_agent()
+    original_shape = registry.load_dataframe("products").shape
+    agent.handle("drop duplicates in products")
+    original_after = registry.load_dataframe("products").shape
+    clean_exists = "products_clean" in registry.list_datasets()
+
+    ok = (original_after == original_shape) and clean_exists
+    print(f"{'[PASS]' if ok else '[FAIL]'} Original unchanged; products_clean created")
+    print(f"       Original shape before : {original_shape}")
+    print(f"       Original shape after  : {original_after}")
+    print(f"       products_clean exists : {clean_exists}\n")
+    passed += ok
+    failed += (not ok)
+
+    restore(); backup()
+
+    # ── CLEANING: DROP DUPLICATES ───────────────────────────
+    print("--- Cleaning: Drop Duplicates ---\n")
+
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df_with_dups = pd.concat([df, df.head(10)], ignore_index=True)
+    registry.update_dataset("products", df_with_dups, extract_schema(df_with_dups))
+
+    run_test(
+        label="Drop 10 injected duplicate rows",
+        query="drop duplicates in products",
+        check_fn=lambda result, ag: (
+            "dropped 10" in result.lower() and
+            ag.registry.load_dataframe("products_clean").duplicated().sum() == 0
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="No duplicates present → reports 0 dropped",
+        query="drop duplicates in products",
+        check_fn=lambda result, ag: "dropped 0" in result.lower(),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── CLEANING: FILL NULLS ────────────────────────────────
+    print("--- Cleaning: Fill Nulls ---\n")
+
+    # symmetric numeric (|skew| < 1) → mean
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df.loc[0:9, "Price"] = None
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Fill symmetric Price column → uses mean",
+        query="fill price in products",
+        check_fn=lambda result, ag: (
+            "mean" in result.lower() and
+            ag.registry.load_dataframe("products_clean")["Price"].isnull().sum() == 0
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # skewed numeric (|skew| >= 1) → median
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df["Price"] = df["Price"].astype(float)
+    df.loc[0:9, "Price"] = None
+    df.loc[10:, "Price"] = df.loc[10:, "Price"] ** 3
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Fill skewed Price column → uses median",
+        query="fill price in products",
+        check_fn=lambda result, ag: (
+            "median" in result.lower() and
+            ag.registry.load_dataframe("products_clean")["Price"].isnull().sum() == 0
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # categorical → mode
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df.loc[0:9, "Category"] = None
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Fill categorical Category column → uses mode",
+        query="fill category in products",
+        check_fn=lambda result, ag: (
+            "mode" in result.lower() and
+            ag.registry.load_dataframe("products_clean")["Category"].isnull().sum() == 0
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # fill all columns at once
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df.loc[0:9, "Price"] = None
+    df.loc[0:4, "Category"] = None
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Fill all nulls across every column in one call",
+        query="fill nulls in products",
+        check_fn=lambda result, ag: (
+            "filled" in result.lower() and
+            ag.registry.load_dataframe("products_clean").isnull().sum().sum() == 0
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # column with no nulls
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Fill column with no nulls → no-op message",
+        query="fill price in products",
+        check_fn=lambda result, ag: "no missing" in result.lower(),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── CLEANING: DROP CONSTANT COLUMNS ────────────────────
+    print("--- Cleaning: Drop Constant Columns ---\n")
+
+    # Currency is constant (USD) in the original products data
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Drop existing constant column (Currency=USD)",
+        query="drop constant columns in products",
+        check_fn=lambda result, ag: (
+            "currency" in result.lower() and
+            "Currency" not in ag.registry.load_dataframe("products_clean").columns
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # inject an additional constant column
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df["TestConst"] = 0
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Drop multiple constant columns (Currency + injected TestConst)",
+        query="drop constant columns in products",
+        check_fn=lambda result, ag: (
+            "testconst" in result.lower() and
+            "TestConst" not in ag.registry.load_dataframe("products_clean").columns and
+            "Currency" not in ag.registry.load_dataframe("products_clean").columns
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── CLEANING: STRIP WHITESPACE ──────────────────────────
+    print("--- Cleaning: Strip Whitespace ---\n")
+
+    registry, agent = fresh_agent()
+    df = registry.load_dataframe("products")
+    df["Name"] = "  " + df["Name"].astype(str) + "  "
+    registry.update_dataset("products", df, extract_schema(df))
+
+    run_test(
+        label="Strip whitespace from string columns",
+        query="strip whitespace in products",
+        check_fn=lambda result, ag: (
+            "stripped" in result.lower() and
+            not ag.registry.load_dataframe("products_clean")["Name"]
+                .str.startswith(" ").any()
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── CLEANING: DROP COLUMN ───────────────────────────────
+    print("--- Cleaning: Drop Column ---\n")
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Drop Description column",
+        query="drop description in products",
+        check_fn=lambda result, ag: (
+            "dropped" in result.lower() and
+            "Description" not in ag.registry.load_dataframe("products_clean").columns
+        ),
+        agent=agent,
+    )
+
+    run_test(
+        label="Drop non-existent column → not found",
+        query="drop ghostcol in products",
+        check_fn=lambda result, ag: "not found" in result.lower(),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── TRANSFORMATIONS ─────────────────────────────────────
+    print("--- Transformations (secondary) ---\n")
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Normalize Price → [0, 1]",
+        query="normalize price in products",
+        check_fn=lambda result, ag: (
+            "normalized" in result.lower() and
+            ag.registry.load_dataframe("products_clean")["Price"].between(0, 1).all()
+        ),
+        agent=agent,
+    )
+
+    run_test(
+        label="Normalize non-numeric column → blocked",
+        query="normalize category in products",
+        check_fn=lambda result, ag: "not numeric" in result.lower(),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Encode Category → integer codes",
+        query="encode category in products",
+        check_fn=lambda result, ag: (
+            "label-encoded" in result.lower() and
+            pd.api.types.is_integer_dtype(
+                ag.registry.load_dataframe("products_clean")["Category"]
+            )
+        ),
+        agent=agent,
+    )
+
+    run_test(
+        label="Encode numeric column → blocked",
+        query="encode price in products",
+        check_fn=lambda result, ag: "not categorical" in result.lower(),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Rename Stock to inventory",
+        query="rename stock to inventory in products",
+        check_fn=lambda result, ag: (
+            "renamed" in result.lower() and
+            "inventory" in ag.registry.load_dataframe("products_clean").columns and
+            "Stock" not in ag.registry.load_dataframe("products_clean").columns
+        ),
+        agent=agent,
+    )
+
+    restore(); backup()
+
+    # ── EDGE CASES ──────────────────────────────────────────
+    print("--- Edge Cases ---\n")
+
+    registry, agent = fresh_agent()
+
+    run_test(
+        label="Unknown operation → fallback message",
+        query="sort price in products",
+        check_fn=lambda result, ag: "not understood" in result.lower(),
+        agent=agent,
+    )
+
+finally:
+    restore()
+
+print("=" * 60)
+print(f"Results: {passed} passed, {failed} failed")
+if failed == 0:
+    print("All tests passed.")
+print("=" * 60)

testing/test_visualization.py

@@ -0,0 +1,134 @@
+import matplotlib
+matplotlib.use("Agg")
+
+import matplotlib.pyplot as plt
+plt.show = lambda: None
+
+import os
+from data.registry import DatasetRegistry
+from agents.visualization_agent import VisualizationAgent
+
+OUTPUT_DIR = "test_output"
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+registry = DatasetRegistry()
+agent = VisualizationAgent(registry)
+
+passed = 0
+failed = 0
+
+
+def run_test(label, query, output_file=None, expect_chart=True, expect_message=None):
+    global passed, failed
+    plt.close("all")
+
+    result = agent.handle(query)
+    figures = [plt.figure(i) for i in plt.get_fignums()]
+
+    if expect_chart:
+        if figures:
+            fig = figures[-1]
+            fig.tight_layout()
+            path = os.path.join(OUTPUT_DIR, output_file)
+            fig.savefig(path, bbox_inches="tight")
+            plt.close("all")
+            print(f"[PASS] {label}")
+            print(f"       Query   : {query}")
+            print(f"       Result  : {result}")
+            print(f"       Saved to: {path}\n")
+            passed += 1
+        else:
+            print(f"[FAIL] {label}")
+            print(f"       Query  : {query}")
+            print(f"       Result : {result}")
+            print(f"       Expected a chart but none was generated.\n")
+            failed += 1
+    else:
+        plt.close("all")
+        ok = (expect_message is None) or (expect_message in result)
+        tag = "[PASS]" if ok else "[FAIL]"
+        print(f"{tag} {label}")
+        print(f"       Query  : {query}")
+        print(f"       Result : {result}\n")
+        if ok:
+            passed += 1
+        else:
+            failed += 1
+
+
+print("=" * 60)
+print("  Visualization Agent Test Suite")
+print("=" * 60)
+print(f"Datasets loaded: {registry.list_datasets()}\n")
+
+# ── Histograms (guardrail does NOT apply) ───────────────────
+print("--- Histograms ---\n")
+
+run_test(
+    label="Histogram – Price (products, 999 unique values, allowed)",
+    query="histogram price in products",
+    output_file="histogram_price_products.png",
+)
+
+run_test(
+    label="Histogram – Stock (products, 999 unique values, allowed)",
+    query="show histogram of stock in products",
+    output_file="histogram_stock_products.png",
+)
+
+# ── Bar charts – under limit (guardrail allows) ─────────────
+print("--- Bar Charts (within limit) ---\n")
+
+run_test(
+    label="Bar chart – Category (products, 34 unique values → allowed)",
+    query="bar chart category in products",
+    output_file="bar_category_products.png",
+)
+
+# ── Bar charts – over limit (guardrail blocks) ──────────────
+print("--- Bar Charts (guardrail triggered) ---\n")
+
+run_test(
+    label="Bar chart – Color (products, 140 unique values → blocked)",
+    query="bar chart color in products",
+    expect_chart=False,
+    expect_message="Too many to visualize meaningfully",
+)
+
+run_test(
+    label="Bar chart – Brand (products, 72263 unique values → blocked)",
+    query="bar chart brand in products",
+    expect_chart=False,
+    expect_message="Too many to visualize meaningfully",
+)
+
+# ── Edge cases ──────────────────────────────────────────────
+print("--- Edge Cases ---\n")
+
+run_test(
+    label="No column specified → helpful message",
+    query="histogram in products",
+    expect_chart=False,
+    expect_message="Column not found",
+)
+
+run_test(
+    label="Unknown dataset → error message",
+    query="histogram price in unknown_dataset",
+    expect_chart=False,
+)
+
+run_test(
+    label="Unsupported chart type → fallback message",
+    query="scatter plot price in products",
+    expect_chart=False,
+    expect_message="not understood",
+)
+
+# ── Summary ─────────────────────────────────────────────────
+print("=" * 60)
+print(f"Results: {passed} passed, {failed} failed")
+if failed == 0:
+    print("All tests passed.")
+print(f"Charts saved in '{OUTPUT_DIR}/'")
+print("=" * 60)

utils/logger.py

@@ -0,0 +1,15 @@
+import logging
+from pathlib import Path
+
+LOG_DIR = Path("logs")
+LOG_DIR.mkdir(exist_ok=True)
+
+LOG_FILE = LOG_DIR / "eda_explorer.log"
+
+logging.basicConfig(
+    filename=LOG_FILE,
+    level=logging.INFO,
+    format="%(asctime)s | %(levelname)s | %(message)s",
+)
+
+logger = logging.getLogger("EDA_EXPLORER")

vector_store/analyze_embeddings.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d51489b7332e27e48fcd6ef7f30efd80310959669fc05597932932e2146a309
+size 104114

vector_store/instruction_embedder.py

@@ -0,0 +1,140 @@
+# from pathlib import Path
+# import pickle
+# from sentence_transformers import SentenceTransformer
+
+# BASE_DIR = Path(__file__).resolve().parent.parent
+
+# pickle_file = BASE_DIR / "vector_store" / "analyze_embeddings.pkl"
+# instruction_file = BASE_DIR / "instructions" / "analyze.txt"
+
+# def embed_analyze_instructions():
+#     instruction_file = BASE_DIR / "instructions" / "analyze.txt"
+#     pickle_file = BASE_DIR / "vector_store" / "analyze_embeddings.pkl"
+
+#     # Ensure directory exists
+#     pickle_file.parent.mkdir(parents=True, exist_ok=True)
+
+#     # If embeddings already exist, load
+#     if pickle_file.exists():
+#         with open(pickle_file, "rb") as f:
+#             data = pickle.load(f)
+#         # print("Analyze embeddings already exist. Loaded from disk.")
+#         return data
+
+#     # Load instructions
+#     with open(instruction_file, "r", encoding="utf-8") as f:
+#         instructions = [line.strip() for line in f if line.strip()]
+
+#     # Embed
+#     model = SentenceTransformer('all-MiniLM-L6-v2')
+#     embeddings = model.encode(instructions)
+#     pickle_file.parent.mkdir(parents=True, exist_ok=True)
+#     # Save
+#     data = {"instructions": instructions, "embeddings": embeddings}
+#     with open(pickle_file, "wb") as f:
+#         pickle.dump(data, f)
+
+#     print(f"Instruction embeddings created and saved: {len(instructions)} instructions")
+#     return data
+
+
+# if __name__ == "__main__":
+#     embed_analyze_instructions()
+
+
+
+
+import os
+import pickle
+import requests
+from pathlib import Path
+from utils.logger import logger
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+PICKLE_FILE = BASE_DIR / "vector_store" / "analyze_embeddings.pkl"
+INSTRUCTION_FILE = BASE_DIR / "instructions" / "analyze.txt"
+
+def get_ollama_embeddings(texts):
+    """Try to get embeddings from local Ollama service."""
+    try:
+        # Default Ollama address
+        url = "http://localhost:11434/api/embed" 
+        # Note: Some Ollama versions use /api/embeddings (plural)
+        embeddings = []
+        for text in texts:
+            response = requests.post(
+                url, 
+                json={"model": "mxbai-embed-large", "input": text}, 
+                timeout=5
+            )
+            embeddings.append(response.json()['embeddings'][0])
+        return embeddings
+    except Exception:
+        return None
+
+def get_hf_api_embeddings(texts):
+    """Try to get embeddings via Hugging Face Inference API."""
+    token = os.environ.get("HF_TOKEN")
+    if not token:
+        return None
+        
+    api_url = "https://api-inference.huggingface.co/pipeline/feature-extraction/sentence-transformers/all-MiniLM-L6-v2"
+    headers = {"Authorization": f"Bearer {token}"}
+    
+    try:
+        response = requests.post(api_url, headers=headers, json={"inputs": texts}, timeout=10)
+        return response.json()
+    except Exception:
+        return None
+
+def embed_analyze_instructions():
+    # 1. Ensure directory exists
+    PICKLE_FILE.parent.mkdir(parents=True, exist_ok=True)
+
+    # 2. Check if cached embeddings exist
+    if PICKLE_FILE.exists():
+        with open(PICKLE_FILE, "rb") as f:
+            return pickle.load(f)
+
+    # 3. Load instructions from file
+    if not INSTRUCTION_FILE.exists():
+        logger.error(f"Instruction file not found at {INSTRUCTION_FILE}")
+        return None
+
+    with open(INSTRUCTION_FILE, "r", encoding="utf-8") as f:
+        instructions = [line.strip() for line in f if line.strip()]
+
+    embeddings = None
+
+    # --- FALLBACK LOGIC ---
+    
+    # Try Ollama First
+    logger.info("Attempting Ollama embeddings...")
+    embeddings = get_ollama_embeddings(instructions)
+
+    # Try HF API Second
+    if embeddings is None:
+        logger.info("Ollama failed. Attempting Hugging Face API...")
+        embeddings = get_hf_api_embeddings(instructions)
+
+    # Local Heavy Fallback Third
+    if embeddings is None:
+        logger.warning("External APIs failed. Loading heavy local SentenceTransformer...")
+        # Lazy import: Only loads Torch/Transformers if absolutely necessary
+        from sentence_transformers import SentenceTransformer
+        model = SentenceTransformer('all-MiniLM-L6-v2')
+        embeddings = model.encode(instructions)
+
+    # 4. Save and Return
+    if embeddings is not None:
+        data = {"instructions": instructions, "embeddings": embeddings}
+        with open(PICKLE_FILE, "wb") as f:
+            pickle.dump(data, f)
+        logger.info(f"Embeddings saved: {len(instructions)} instructions")
+        return data
+    
+    logger.error("Failed to generate embeddings via any method.")
+    return None
+
+if __name__ == "__main__":
+    embed_analyze_instructions()