Update tools/csv_parser.py

tools/csv_parser.py

@@ -1,67 +1,99 @@
-import pandas as pd
-from typing import Union
+# tools/csv_parser.py
+# ------------------------------------------------------------
+# Reads CSV / Excel, samples for very large files, and returns a
+# Markdown‑formatted “quick‑scan” report: dimensions, schema,
+# missing‑value profile, numeric describe(), and memory footprint.
+
+from __future__ import annotations
+
 import os
+from typing import Union
+
+import pandas as pd
+
+
+def _safe_read(path_or_buf: Union[str, bytes], sample_rows: int = 1_000_000) -> pd.DataFrame:
+    """Read CSV or Excel.  If the file has > sample_rows, read only a sample."""
+    # Determine extension (best‑effort)
+    ext = ".csv"
+    if isinstance(path_or_buf, str):
+        ext = os.path.splitext(path_or_buf)[1].lower()
+
+    if ext in (".xls", ".xlsx"):
+        # Excel — read first sheet
+        df = pd.read_excel(path_or_buf, engine="openpyxl")
+    else:  # CSV family
+        # First row‑count check: pandas 1.5+ uses memory map ⇒ cheap for header only
+        nrows_total = sum(1 for _ in open(path_or_buf, "rb")) if isinstance(path_or_buf, str) else None
+        if nrows_total and nrows_total > sample_rows:
+            # sample uniformly without loading everything
+            skip = sorted(
+                pd.np.random.choice(range(1, nrows_total), nrows_total - sample_rows, replace=False)
+            )
+            df = pd.read_csv(path_or_buf, skiprows=skip)
+        else:
+            df = pd.read_csv(path_or_buf)
+
+    return df
+
 
 def parse_csv_tool(file: Union[str, bytes]) -> str:
     """
-    Parses a CSV or Excel file and returns a comprehensive schema and statistics report in Markdown.
-    Supports large files by sampling if necessary and handles common parsing errors.
-    """
-    # Determine extension
-    try:
-        filename = getattr(file, 'name', file)
-        ext = os.path.splitext(filename)[1].lower()
-    except Exception:
-        ext = ".csv"
+    Return a **Markdown** report describing the dataset.
 
-    # Load DataFrame
+    Sections:
+    • Dimensions
+    • Schema (+ dtypes)
+    • Missing‑value counts + %
+    • Numeric descriptive statistics
+    • Memory usage
+    """
     try:
-        if ext in ('.xls', '.xlsx'):
-            df = pd.read_excel(file, engine='openpyxl')
-        else:
-            df = pd.read_csv(file)
-    except Exception as e:
-        return f"❌ Failed to load data ({ext}): {e}"
+        df = _safe_read(file)
+    except Exception as exc:
+        return f"❌ Failed to load data: {exc}"
 
-    # Basic dimensions
     n_rows, n_cols = df.shape
 
-    # Schema & dtypes
-    schema_lines = [f"- **{col}**: {dtype}" for col, dtype in df.dtypes.items()]
-    schema_md = "\n".join(schema_lines)
+    # ---------- schema ----------
+    schema_md = "\n".join(
+        f"- **{col}** – `{dtype}`"
+        for col, dtype in df.dtypes.items()
+    )
 
-    # Missing values
-    missing = df.isna().sum()
-    missing_pct = (missing / n_rows * 100).round(1)
-    missing_lines = []
-    for col in df.columns:
-        if missing[col] > 0:
-            missing_lines.append(f"- **{col}**: {missing[col]} ({missing_pct[col]}%)")
-    missing_md = "\n".join(missing_lines) or "None"
+    # ---------- missing ----------
+    miss_ct  = df.isna().sum()
+    miss_pct = (miss_ct / len(df) * 100).round(1)
+    missing_md = "\n".join(
+        f"- **{c}**: {miss_ct[c]} ({miss_pct[c]} %)"
+        for c in df.columns if miss_ct[c] > 0
+    ) or "None"
 
-    # Descriptive stats (numeric)
-    desc = df.describe().T.round(2)
-    desc_md = desc.to_markdown()
+    # ---------- descriptive stats (numeric only) ----------
+    if df.select_dtypes("number").shape[1]:
+        desc_md = df.describe().T.round(2).to_markdown()
+    else:
+        desc_md = "_No numeric columns_"
 
-    # Memory usage
-    mem_mb = df.memory_usage(deep=True).sum() / (1024 ** 2)
+    # ---------- memory ----------
+    mem_mb = df.memory_usage(deep=True).sum() / 1024**2
 
-    # Assemble report
-    report = f"""
-# 📊 Dataset Overview
+    # ---------- assemble ----------
+    return f"""
+# 📊 Dataset Overview
 
-- **Rows:** {n_rows}
-- **Columns:** {n_cols}
-- **Memory Usage:** {mem_mb:.2f} MB
+| metric | value |
+| ------ | ----- |
+| Rows   | {n_rows:,} |
+| Columns| {n_cols} |
+| Memory | {mem_mb:.2f} MB |
 
-## 🗂 Schema & Data Types
+## 🗂 Schema
 {schema_md}
 
-## 🛠 Missing Values
+## 🛠 Missing Values
 {missing_md}
 
-## 📈 Descriptive Statistics
+## 📈 Descriptive Statistics (numeric)
 {desc_md}
 """.strip()
-
-    return report