Deploy DataPilot AI production Docker Space

.dockerignore

@@ -0,0 +1,25 @@
+.git
+.github
+.venv
+venv
+__pycache__
+*.py[cod]
+.pytest_cache
+.ruff_cache
+.coverage
+htmlcov
+artifacts
+tests
+docs
+scripts
+worker
+api
+*.db
+*.sqlite
+*.sqlite3
+.env
+.streamlit/secrets.toml
+Dockerfile.api
+Dockerfile.worker
+docker-compose.yml
+render.yaml

.env.example

@@ -0,0 +1,18 @@
+ENVIRONMENT=development
+ARTIFACT_ROOT=artifacts
+DATABASE_URL=sqlite:///artifacts/datapilot.db
+MAX_UPLOAD_MB=25
+MAX_ROWS=100000
+MAX_COLUMNS=250
+MAX_CATEGORIES_PER_FEATURE=100
+MAX_ENCODED_FEATURES=5000
+API_KEY=
+REQUESTS_PER_MINUTE=30
+MAX_CRITIC_RETRIES=1
+ENABLE_MLFLOW=false
+MLFLOW_TRACKING_URI=file:./artifacts/mlruns
+# Optional: the deterministic application works without an LLM key.
+GEMINI_API_KEY=
+GEMINI_MODEL=gemini-2.5-flash
+CORS_ORIGINS=http://localhost:8501
+

.github/CODEOWNERS

@@ -0,0 +1,5 @@
+* @dineshbarri
+/.github/ @dineshbarri
+/datapilot/ @dineshbarri
+/api/ @dineshbarri
+/docs/ @dineshbarri

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,20 @@
+name: Bug report
+description: Report a reproducible DataPilot defect
+title: "[Bug]: "
+labels: [bug]
+body:
+  - type: textarea
+    id: description
+    attributes: {label: Description, description: What happened?}
+    validations: {required: true}
+  - type: textarea
+    id: reproduce
+    attributes: {label: Reproduction, description: Provide minimal safe reproduction steps.}
+    validations: {required: true}
+  - type: input
+    id: version
+    attributes: {label: Version or commit}
+    validations: {required: true}
+  - type: textarea
+    id: logs
+    attributes: {label: Sanitized logs, description: Remove API keys and private data.}

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,16 @@
+name: Feature request
+description: Propose a product, ML, platform, or evaluation improvement
+title: "[Feature]: "
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes: {label: Problem, description: Which user or engineering problem should this solve?}
+    validations: {required: true}
+  - type: textarea
+    id: proposal
+    attributes: {label: Proposed approach}
+    validations: {required: true}
+  - type: textarea
+    id: risks
+    attributes: {label: Privacy, security, and ML risks}

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule: {interval: weekly}
+    groups:
+      python-dependencies: {patterns: ["*"]}
+  - package-ecosystem: docker
+    directory: /
+    schedule: {interval: weekly}
+  - package-ecosystem: github-actions
+    directory: /
+    schedule: {interval: weekly}

.github/pull_request_template.md

@@ -0,0 +1,15 @@
+## Summary
+
+## Why this change
+
+## Validation
+
+- [ ] Tests added or updated
+- [ ] `ruff check .` passes
+- [ ] `pytest --cov` passes
+- [ ] Security/privacy impact reviewed
+- [ ] Documentation updated
+
+## ML/AI impact
+
+Describe changes to data handling, evaluation methodology, prompts, models, or metrics.

.github/workflows/ci.yml

@@ -0,0 +1,121 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  security-events: write
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+      - run: python -m pip install --upgrade pip && pip install -e ".[dev]"
+      - run: ruff check .
+      - run: ruff format --check .
+      - run: pytest --cov=datapilot --cov=api --cov-report=term-missing --cov-report=xml --cov-fail-under=75
+      - run: python -m compileall -q datapilot api worker app.py streamlit_app.py
+      - name: Generate example model card
+        if: matrix.python-version == '3.12'
+        run: python scripts/generate_example_model_card.py
+      - uses: actions/upload-artifact@v4
+        if: matrix.python-version == '3.12'
+        with:
+          name: evaluation-evidence
+          path: |
+            coverage.xml
+            build/example-model-card.md
+
+  package:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: pip install build twine
+      - run: python -m build
+      - run: twine check dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: python-package
+          path: dist/
+
+  dependency-audit:
+   runs-on: ubuntu-latest
+   steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+      with:
+        python-version: "3.12"
+        cache: pip
+    - run: python -m pip install --upgrade pip pip-audit
+    - run: pip-audit --requirement requirements.txt
+
+  containers:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - file: Dockerfile
+            image: datapilot-ui
+          - file: Dockerfile.api
+            image: datapilot-api
+          - file: Dockerfile.worker
+            image: datapilot-worker
+    steps:
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.file }}
+          push: false
+          load: true
+          tags: ${{ matrix.image }}:ci
+      - uses: aquasecurity/trivy-action@v0.36.0
+        with:
+          image-ref: ${{ matrix.image }}:ci
+          format: sarif
+          output: "trivy-${{ matrix.image }}.sarif"
+          severity: HIGH,CRITICAL
+          ignore-unfixed: true
+      - uses: github/codeql-action/upload-sarif@v4
+        if: github.actor != 'dependabot[bot]'
+        with:
+          sarif_file: "trivy-${{ matrix.image }}.sarif"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: "trivy-${{ matrix.image }}"
+          path: "trivy-${{ matrix.image }}.sarif"
+
+  secrets:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: gitleaks/gitleaks-action@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,23 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+permissions:
+  contents: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with: {python-version: "3.12"}
+      - run: pip install build
+      - run: python -m build
+      - run: git archive --format=zip --output=dist/DataPilot-AI-${GITHUB_REF_NAME}.zip HEAD
+      - uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+          files: dist/*

.gitignore

@@ -0,0 +1,24 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+build/
+dist/
+.venv/
+venv/
+.env
+artifacts/
+mlruns/
+*.joblib
+*.parquet
+.DS_Store
+.idea/
+.vscode/
+
+.streamlit/secrets.toml
+*.db
+*.sqlite
+*.sqlite3
+

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.12.8
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+      - id: check-added-large-files
+        args: [--maxkb=5000]
+      - id: detect-private-key
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.28.0
+    hooks:
+      - id: gitleaks

.streamlit/config.toml

@@ -0,0 +1,11 @@
+[theme]
+primaryColor = "#6045D8"
+backgroundColor = "#FBFAFF"
+secondaryBackgroundColor = "#F4F1FF"
+textColor = "#172033"
+font = "sans serif"
+
+[server]
+headless = true
+maxUploadSize = 25
+

CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+
+All notable changes follow [Keep a Changelog](https://keepachangelog.com/) and semantic versioning.
+
+## [2.0.0] - 2026-08-10
+
+### Added
+
+- Training-only cross-validation selection and one-time untouched test evaluation.
+- Sparse, cardinality-bounded categorical encoding and encoded-width guardrails.
+- Structured candidate failure records.
+- Asynchronous API job contract, cancellation, authentication, rate limiting, and correlation IDs.
+- CI, Dependabot, pre-commit, Docker context controls, templates, threat model, privacy, and evaluation docs.
+
+### Changed
+
+- Consolidated Streamlit deployment onto the `app.py` implementation.
+- Sanitized API and job errors returned to clients.

CONTRIBUTING.md

@@ -0,0 +1,13 @@
+# Contributing
+
+1. Create a feature branch.
+2. Install `pip install -e ".[dev]"`.
+3. Run `ruff check .`.
+4. Run `pytest --cov=datapilot --cov=api`.
+5. Keep computed metrics deterministic and source-backed.
+6. Add a test for every new quality rule, workflow route or API behavior.
+7. Never commit datasets containing personal data, credentials or generated artifacts.
+
+Pull requests should explain the user problem, architecture impact, validation performed and
+any new security or model-risk considerations.
+

Dockerfile

@@ -0,0 +1,32 @@
+FROM python:3.12-slim AS runtime
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    STREAMLIT_SERVER_HEADLESS=true \
+    STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
+
+WORKDIR /app
+
+RUN addgroup --system datapilot \
+    && adduser --system --ingroup datapilot --home /home/datapilot datapilot
+
+COPY requirements.txt ./requirements.txt
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir -r requirements.txt
+
+COPY app.py streamlit_app.py ./
+COPY datapilot ./datapilot
+COPY .streamlit ./.streamlit
+
+RUN mkdir -p /app/artifacts /home/datapilot/.streamlit \
+    && chown -R datapilot:datapilot /app /home/datapilot
+
+USER datapilot
+
+EXPOSE 7860
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:7860/_stcore/health', timeout=3)"
+
+CMD ["streamlit", "run", "streamlit_app.py", "--server.address=0.0.0.0", "--server.port=7860", "--server.headless=true", "--browser.gatherUsageStats=false"]

Dockerfile.api

@@ -0,0 +1,16 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
+WORKDIR /app
+RUN addgroup --system datapilot && adduser --system --ingroup datapilot datapilot
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY datapilot ./datapilot
+COPY api ./api
+RUN mkdir -p /app/artifacts && chown -R datapilot:datapilot /app
+USER datapilot
+EXPOSE 8000
+HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
+  CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"
+CMD ["uvicorn", "api.main:app", "--host=0.0.0.0", "--port=8000"]
+

Dockerfile.worker

@@ -0,0 +1,13 @@
+FROM python:3.12-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
+WORKDIR /app
+RUN addgroup --system datapilot && adduser --system --ingroup datapilot datapilot
+RUN pip install --no-cache-dir fastapi uvicorn pydantic
+COPY datapilot/safety.py ./datapilot/safety.py
+COPY datapilot/__init__.py ./datapilot/__init__.py
+COPY worker ./worker
+USER datapilot
+EXPOSE 8010
+CMD ["uvicorn", "worker.main:app", "--host=0.0.0.0", "--port=8010"]
+

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026 Dinesh Barri
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

Makefile

@@ -0,0 +1,20 @@
+.PHONY: install test lint format coverage build run api security
+
+install:
+	python -m pip install -e ".[dev]"
+test:
+	python -m pytest
+lint:
+	python -m ruff check .
+format:
+	python -m ruff format .
+coverage:
+	python -m pytest --cov=datapilot --cov=api --cov-fail-under=75
+build:
+	python -m build
+run:
+	python -m streamlit run app.py
+api:
+	python -m uvicorn api.main:app --reload
+security:
+	python -m pip_audit

README.md

@@ -1,11 +1,361 @@
 ---
 title: DataPilot AI Agent
-emoji: 📊
-colorFrom: red
+emoji: "📊"
+colorFrom: indigo
 colorTo: green
 sdk: docker
-pinned: false
+app_port: 7860
+pinned: true
 license: mit
+short_description: Evidence-grounded autonomous data science and ML copilot
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+<div align="center">
+
+<img src="https://capsule-render.vercel.app/api?type=rect&color=0:21154F,48:6045D8,100:19A98E&height=190&section=header&text=DataPilot%20AI&fontSize=48&fontColor=ffffff&animation=fadeIn&desc=Autonomous%20Data%20Science%20%C2%B7%20Evidence-Grounded%20ML%20%C2%B7%20Explainable%20Decisions&descSize=17&descAlignY=72" width="100%" alt="DataPilot AI banner">
+
+<p>
+  <img src="https://readme-typing-svg.herokuapp.com/?font=Fira+Code&size=16&duration=2600&pause=850&color=6D4AFF&center=true&vCenter=true&width=780&lines=CSV+or+Parquet+%E2%86%92+Quality+Audit+%E2%86%92+Validated+Model;8+Specialized+Agents+%7C+Stateful+LangGraph+Orchestration;Leakage-Safe+Pipelines+%7C+Critic+Retry+Loop+%7C+SHAP-Ready;Streamlit+%7C+FastAPI+%7C+DuckDB+%7C+scikit-learn" alt="DataPilot AI capabilities">
+</p>
+
+[![Python](https://img.shields.io/badge/Python-3.11--3.13-3776AB?style=for-the-badge&logo=python&logoColor=white)](https://www.python.org/)
+[![LangGraph](https://img.shields.io/badge/LangGraph-Stateful_Agents-1C1C1C?style=for-the-badge)](https://langchain-ai.github.io/langgraph/)
+[![Streamlit](https://img.shields.io/badge/Streamlit-Recruiter_Demo-FF4B4B?style=for-the-badge&logo=streamlit&logoColor=white)](https://datapilot-ai-agent.streamlit.app/)
+[![FastAPI](https://img.shields.io/badge/FastAPI-Production_API-009688?style=for-the-badge&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/)
+[![CI](https://github.com/dineshbarri/DataPilot-AI/actions/workflows/ci.yml/badge.svg)](https://github.com/dineshbarri/DataPilot-AI/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/License-MIT-F5C518?style=for-the-badge)](LICENSE)
+
+**[API Docs](#fastapi)** ·
+**[Architecture](docs/ARCHITECTURE.md)** ·
+**[Security](docs/SECURITY.md)** ·
+**[Model Governance](docs/MODEL_GOVERNANCE.md)** ·
+**[Benchmarks](docs/BENCHMARKS.md)** ·
+**[API Examples](docs/API_EXAMPLES.md)**
+
+</div>
+
+---
+
+## Why DataPilot AI?
+
+Most “AI data scientist” demos upload a CSV, run preprocessing on the entire dataset, compare
+a few models, and ask an LLM to write an impressive-sounding summary. That is fast—but it can
+leak test information, exaggerate confidence, and produce insights with no numerical provenance.
+
+**DataPilot AI treats trust as a feature.** It coordinates a stateful agent team that audits the
+data, plans the experiment, builds leakage-safe pipelines, compares models, challenges the
+winner, explains predictive signals, and exports reproducible artifacts. Every displayed metric
+comes from deterministic computation. The optional LLM may improve wording; it cannot create
+new numbers or execute code.
+
+### Recruiter five-minute test
+
+1. Open the app.
+2. Keep **Iris classification** selected.
+3. Click **Run autonomous analysis**.
+4. Inspect the model comparison, critic decision, feature importance and full agent trace.
+5. Download the fitted pipeline, model card and standalone report.
+
+No account, upload or API key is required.
+
+---
+
+## Product capabilities
+
+| Stage | What DataPilot does | Evidence produced |
+|---|---|---|
+| Data intake | Accepts bounded CSV, TSV, Excel, JSON, Parquet or packaged demos | row/column limits and validated schema |
+| Data quality | Detects missingness, duplicates, target gaps, imbalance, outliers and leakage-like fields | evidence registry with IDs, source and method |
+| EDA | Uses DuckDB and pandas for compact profiles, cardinality and correlations | dataset profile and statistical summary |
+| Planning | Infers classification/regression, primary metric, validation strategy and risk controls | typed Pydantic analysis plan |
+| Feature engineering | Builds numeric and categorical transformers inside the model pipeline | transformation plan and fitted pipeline |
+| Modeling | Selects linear, forest, extra-trees and optional XGBoost models by training-only CV | CV mean/stability and one-time untouched test results |
+| Evaluation | Applies thresholds and validation-consistency checks | critic approval, rejection reasons and retry count |
+| Explainability | Uses SHAP only when the optional dependency and fitted estimator are compatible; otherwise permutation importance | ranked predictive signals and caveats |
+| Reporting | Produces a dashboard, evidence-backed narrative and portable artifacts | HTML report, model card, JSON, joblib pipeline |
+| Follow-up | Answers questions from persisted run evidence | bounded, non-hallucinatory responses |
+
+---
+
+## Real agentic orchestration
+
+The “agents” are typed LangGraph nodes combining deterministic Python computation with optional,
+evidence-bounded LLM narration. The critic controls a conditional edge:
+weak or unstable analysis returns to the modeling node before explanation is allowed.
+
+```mermaid
+flowchart LR
+    A["CSV / Parquet / Demo"] --> B["Data Quality Agent"]
+    B --> C["EDA Agent"]
+    C --> D["Statistical Agent"]
+    D --> E["Planning Agent"]
+    E --> F["Feature Engineering Agent"]
+    F --> G["Modeling Agent"]
+    G --> H{"Evaluation / Critic"}
+    H -->|"Reject + retry"| G
+    H -->|"Approve"| I["Explainability Agent"]
+    I --> J["Executive Insights Agent"]
+    J --> K["Report · Model Card · Pipeline · Evidence"]
+```
+
+The Streamlit **Agent trace** tab shows every completed node, its duration and decision.
+
+---
+
+## Leakage-safe ML design
+
+```python
+pipeline = Pipeline(
+    [
+        ("preprocessor", ColumnTransformer(...)),
+        ("model", candidate_model),
+    ]
+)
+
+# Imputers, encoders and scalers learn only from training folds.
+pipeline.fit(x_train, y_train)
+```
+
+- Split occurs before learned preprocessing.
+- Cross-validation refits the complete pipeline in every fold.
+- Classification defaults to balanced accuracy and stratification where possible.
+- Candidates are selected only by training-partition CV; the selected model touches the test set once.
+- Target-like names and identifier cardinality are flagged for human review.
+- Predictive importance is never described as causality.
+
+---
+
+## Dashboard experience
+
+The Streamlit application is summary-first and useful before any upload:
+
+- **Built-in demos:** Iris, Breast Cancer and Diabetes Progression.
+- **Executive overview:** findings, recommendations and quality risks.
+- **Model laboratory:** candidate comparison, CV stability and critic gate.
+- **Explainability:** interactive Plotly feature-importance view.
+- **Agent trace:** visible orchestration and retry behavior.
+- **Artifacts and Q&A:** downloads, evidence registry and run-specific questions.
+
+![DataPilot production architecture](docs/architecture.svg)
+
+The public demo URL is intentionally not claimed until a monitored deployment exists. Add a
+real browser capture under `assets/` together with the deployment URL after release validation.
+
+---
+
+## Technology stack
+
+| Layer | Technology |
+|---|---|
+| Agent orchestration | LangGraph typed state and conditional routing |
+| User interface | Streamlit and Plotly |
+| API | FastAPI and Pydantic |
+| Analytical engine | DuckDB, pandas and NumPy |
+| ML | scikit-learn; optional XGBoost and Optuna |
+| Explainability | optional SHAP with permutation fallback |
+| Persistence | SQLAlchemy; SQLite locally, PostgreSQL in production |
+| Experiment tracking | structured agent trace; optional MLflow/OpenTelemetry |
+| Artifacts | local storage interface, ready for S3/MinIO replacement |
+| Delivery | Docker Compose, Render Blueprint and Streamlit Cloud |
+| Quality | Pytest, Ruff, coverage, compile checks and Docker builds in GitHub Actions |
+
+---
+
+## Repository structure
+
+```text
+DataPilot-AI/
+├── datapilot/
+│   ├── config.py             # typed environment configuration and limits
+│   ├── data.py               # safe readers, DuckDB overview and sample datasets
+│   ├── quality.py            # quality, leakage, imbalance, outlier and drift checks
+│   ├── modeling.py           # leakage-safe pipelines and model comparison
+│   ├── workflow.py           # LangGraph agent graph and critic loop
+│   ├── insights.py           # deterministic + optional evidence-bounded narrative
+│   ├── reports.py            # HTML report, model card, pipeline and JSON export
+│   ├── persistence.py        # SQL run store and artifact interface
+│   ├── observability.py      # optional MLflow integration
+│   └── safety.py             # expression-only AST security policy
+├── api/main.py               # versioned FastAPI application
+├── worker/main.py            # optional isolated calculation worker
+├── tests/                    # quality, safety, API and end-to-end workflow tests
+├── docs/                     # architecture, deployment, security and governance
+├── app.py                    # canonical premium Streamlit implementation
+├── streamlit_app.py          # Streamlit Cloud compatibility shim
+├── Dockerfile*               # non-root UI, API and worker images
+├── docker-compose.yml        # constrained local multi-service stack
+├── render.yaml               # Render API deployment blueprint
+└── .github/workflows/ci.yml  # lint, tests, coverage, compile and image builds
+```
+
+---
+
+## Quick start
+
+### Local Streamlit demo
+
+```bash
+git clone https://github.com/dineshbarri/DataPilot-AI.git
+cd DataPilot-AI
+
+python -m venv .venv
+# Windows
+.venv\Scripts\activate
+# macOS/Linux
+source .venv/bin/activate
+
+pip install -e ".[dev]"
+streamlit run app.py
+```
+
+For the validated Python 3.12 reference environment, use `pip install -r requirements.lock`.
+The lock snapshot is refreshed after dependency updates and tested across supported interpreters.
+
+Open <http://localhost:8501>.
+
+### Full AI/AutoML/observability extras
+
+```bash
+pip install -e ".[all,dev]"
+```
+
+The default installation intentionally stays deployable on modest public-demo infrastructure.
+
+### FastAPI
+
+```bash
+uvicorn api.main:app --reload --port 8000
+```
+
+Open <http://localhost:8000/docs>.
+
+Example:
+
+```bash
+curl -X POST http://localhost:8000/v1/analyze/sample \
+  -H "Content-Type: application/json" \
+  -d "{\"sample\":\"iris\"}"
+```
+
+The API returns `202 Accepted` with a job ID. Poll `GET /v1/jobs/{job_id}` and cancel queued
+work with `DELETE /v1/jobs/{job_id}`. Set `API_KEY` in deployed environments and send it as
+`X-API-Key`. The bundled in-process queue is for development; use the production topology in
+`docs/ARCHITECTURE.md` for durable execution.
+
+### Docker
+
+```bash
+docker compose up --build
+```
+
+The optional worker runs with no network, a read-only filesystem, dropped capabilities,
+memory/CPU/PID limits and expression-only AST validation.
+
+---
+
+## Configuration
+
+Copy `.env.example` to `.env`.
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `DATABASE_URL` | SQLite | Set a PostgreSQL URL for persistent production runs |
+| `ARTIFACT_ROOT` | `artifacts` | Root for reports, pipelines and model cards |
+| `MAX_UPLOAD_MB` | `25` | Public upload protection |
+| `MAX_ROWS` | `100000` | Maximum rows per analysis |
+| `MAX_COLUMNS` | `250` | Maximum feature width |
+| `MAX_CATEGORIES_PER_FEATURE` | `100` | Bound categorical expansion |
+| `MAX_ENCODED_FEATURES` | `5000` | Refuse unsafe estimated encoded width |
+| `API_KEY` | empty | Optional API authentication; required for public deployment |
+| `REQUESTS_PER_MINUTE` | `30` | Per-client API rate limit |
+| `MAX_CRITIC_RETRIES` | `1` | Conditional modeling retry budget |
+| `ENABLE_MLFLOW` | `false` | Enable optional experiment tracking |
+| `GEMINI_API_KEY` | empty | Optional narrative refinement only |
+
+No credential is embedded in the repository.
+
+---
+
+## Testing and engineering quality
+
+```bash
+pip install -e ".[dev]"
+ruff check .
+pytest --cov=datapilot --cov=api --cov-fail-under=75
+python -m compileall datapilot api worker app.py streamlit_app.py
+```
+
+CI runs Python 3.11, 3.12 and 3.13, enforces coverage, builds the package and all three images,
+audits dependencies, scans containers with Trivy, and performs secret detection.
+
+---
+
+## Deployment
+
+### Streamlit Community Cloud
+
+- Entrypoint: `streamlit_app.py`
+- Python: 3.12
+- Secrets: none required; `GEMINI_API_KEY` is optional
+- Default recruiter path: bundled demo dataset
+
+### Render / Railway / Fly.io
+
+Deploy `Dockerfile.api`, attach PostgreSQL, and configure durable object storage if artifacts
+must survive container replacement. See [the deployment guide](docs/DEPLOYMENT.md).
+
+### Production recommendation
+
+The included in-process job manager makes local requests non-blocking. For durable production
+jobs, replace it with a queue and separate workers, PostgreSQL state, S3-compatible artifacts,
+and short-lived sandboxed workers for any future code-execution capability.
+
+---
+
+## Responsible-use boundaries
+
+DataPilot is an exploratory decision-support system, not an automatic production approval
+authority. Before consequential use, complete:
+
+- target and leakage review
+- out-of-time and segment evaluation
+- privacy and retention assessment
+- fairness and disparate-impact evaluation
+- domain and legal approval
+- monitoring, rollback and retraining ownership
+
+See [Model Governance](docs/MODEL_GOVERNANCE.md) and [Security](docs/SECURITY.md).
+
+---
+
+## Roadmap
+
+- [ ] Background job queue and live progress streaming
+- [ ] S3/MinIO artifact adapter with signed downloads
+- [ ] Native PostgreSQL checkpoints for resumable LangGraph runs
+- [ ] Optuna study dashboard and experiment comparison
+- [ ] Time-series and clustering task families
+- [ ] Fairness and segment-performance report
+- [ ] Data-contract and schema-drift registry
+- [ ] Authenticated multi-tenant workspace
+
+---
+
+## Creator
+
+### Dinesh Barri
+
+AI Engineer building agentic systems, data products, RAG applications and production-oriented
+machine-learning workflows.
+
+[![GitHub](https://img.shields.io/badge/GitHub-dineshbarri-181717?style=for-the-badge&logo=github)](https://github.com/dineshbarri)
+[![LinkedIn](https://img.shields.io/badge/LinkedIn-Dinesh_Barri-0A66C2?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/in/dinesh-barri-7654b010b)
+
+---
+
+## License
+
+Released under the [MIT License](LICENSE).
+
+If this project helps you, please star the repository and share the live demo.
+

SECURITY.md

@@ -0,0 +1,11 @@
+# Security policy
+
+Supported security fixes target the latest release and the `main` branch.
+
+Do not open public issues for vulnerabilities or include datasets, credentials, API keys, or
+database URLs in reports. Report vulnerabilities privately through GitHub Security Advisories:
+
+<https://github.com/dineshbarri/DataPilot-AI/security/advisories/new>
+
+Include affected version, reproduction steps, impact, and a suggested mitigation when possible.
+See [docs/SECURITY.md](docs/SECURITY.md) and [docs/THREAT_MODEL.md](docs/THREAT_MODEL.md).

api/__init__.py

@@ -0,0 +1 @@
+"""FastAPI service for DataPilot AI."""

api/main.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import logging
+import time
+from collections import defaultdict, deque
+from pathlib import Path
+from uuid import uuid4
+
+from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from datapilot.config import get_settings
+from datapilot.data import load_sample, read_dataset
+from datapilot.insights import answer_follow_up
+from datapilot.jobs import JobManager
+from datapilot.persistence import RunStore
+from datapilot.workflow import run_analysis
+
+settings = get_settings()
+logger = logging.getLogger(__name__)
+jobs = JobManager(workers=2)
+request_windows: dict[str, deque[float]] = defaultdict(deque)
+app = FastAPI(
+    title="DataPilot AI API",
+    version="1.0.0",
+    description="Evidence-grounded autonomous data science with LangGraph.",
+)
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=[item.strip() for item in settings.cors_origins.split(",")],
+    allow_methods=["GET", "POST"],
+    allow_headers=["*"],
+)
+
+
+@app.middleware("http")
+async def security_middleware(request: Request, call_next):
+    correlation_id = request.headers.get("X-Correlation-ID") or uuid4().hex
+    request.state.correlation_id = correlation_id
+    if request.url.path.startswith("/v1/"):
+        if settings.api_key and request.headers.get("X-API-Key") != settings.api_key:
+            return _error_response(401, "Authentication required.", correlation_id)
+        client = request.client.host if request.client else "unknown"
+        now = time.monotonic()
+        window = request_windows[client]
+        while window and now - window[0] > 60:
+            window.popleft()
+        if len(window) >= settings.requests_per_minute:
+            return _error_response(429, "Rate limit exceeded.", correlation_id)
+        window.append(now)
+    response = await call_next(request)
+    response.headers["X-Correlation-ID"] = correlation_id
+    return response
+
+
+def _error_response(status_code: int, message: str, correlation_id: str):
+    from fastapi.responses import JSONResponse
+
+    return JSONResponse(
+        status_code=status_code,
+        content={"detail": message, "correlation_id": correlation_id},
+        headers={"X-Correlation-ID": correlation_id},
+    )
+
+
+def _safe_failure(exc: Exception, correlation_id: str) -> HTTPException:
+    logger.exception(
+        "Analysis failure correlation_id=%s category=%s", correlation_id, type(exc).__name__
+    )
+    return HTTPException(
+        status_code=500,
+        detail={
+            "message": "Analysis failed. Use the correlation ID when contacting support.",
+            "correlation_id": correlation_id,
+        },
+    )
+
+
+class SampleRequest(BaseModel):
+    sample: str
+
+
+class ChatRequest(BaseModel):
+    question: str
+
+
+@app.get("/")
+def root() -> dict[str, str]:
+    return {"name": settings.app_name, "status": "ready", "docs": "/docs"}
+
+
+@app.get("/health")
+def health() -> dict[str, object]:
+    return {
+        "status": "healthy",
+        "environment": settings.environment,
+        "limits": {
+            "max_upload_mb": settings.max_upload_mb,
+            "max_rows": settings.max_rows,
+            "max_columns": settings.max_columns,
+        },
+    }
+
+
+@app.post("/v1/analyze/sample", status_code=202)
+def analyze_sample(request: SampleRequest, http_request: Request):
+    try:
+        frame, target, dataset_name = load_sample(request.sample)
+        job = jobs.submit(lambda: run_analysis(frame, target, dataset_name, settings))
+        return {"job_id": job.job_id, "status": job.status, "status_url": f"/v1/jobs/{job.job_id}"}
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise _safe_failure(exc, http_request.state.correlation_id) from exc
+
+
+@app.post("/v1/analyze/upload")
+async def analyze_upload(
+    request: Request,
+    file: UploadFile = File(...),
+    target: str = Form(...),
+):
+    try:
+        content = await file.read(settings.max_upload_mb * 1024 * 1024 + 1)
+        frame = read_dataset(content, file.filename or "dataset.csv", settings)
+        job = jobs.submit(
+            lambda: run_analysis(frame, target, file.filename or "uploaded_dataset", settings)
+        )
+        return {"job_id": job.job_id, "status": job.status, "status_url": f"/v1/jobs/{job.job_id}"}
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc)) from exc
+    except Exception as exc:
+        raise _safe_failure(exc, request.state.correlation_id) from exc
+
+
+@app.get("/v1/jobs/{job_id}")
+def get_job(job_id: str):
+    job = jobs.get(job_id)
+    if job is None:
+        raise HTTPException(status_code=404, detail="Job not found.")
+    return job.public()
+
+
+@app.delete("/v1/jobs/{job_id}")
+def cancel_job(job_id: str):
+    if jobs.get(job_id) is None:
+        raise HTTPException(status_code=404, detail="Job not found.")
+    return {"job_id": job_id, "cancelled": jobs.cancel(job_id)}
+
+
+@app.get("/v1/runs")
+def recent_runs():
+    return RunStore(settings).list_recent()
+
+
+@app.get("/v1/runs/{run_id}")
+def get_run(run_id: str):
+    run = RunStore(settings).get(run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="Run not found.")
+    return run
+
+
+@app.post("/v1/runs/{run_id}/chat")
+def chat_with_run(run_id: str, request: ChatRequest):
+    run = RunStore(settings).get(run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="Run not found.")
+    return {"answer": answer_follow_up(run, request.question)}
+
+
+@app.get("/v1/runs/{run_id}/artifacts/{artifact_name}")
+def download_artifact(run_id: str, artifact_name: str):
+    run = RunStore(settings).get(run_id)
+    if run is None:
+        raise HTTPException(status_code=404, detail="Run not found.")
+    path_string = run.get("artifacts", {}).get(artifact_name)
+    if not path_string:
+        raise HTTPException(status_code=404, detail="Artifact not found.")
+    path = Path(path_string).resolve()
+    artifact_root = settings.artifact_root.resolve()
+    if artifact_root not in path.parents or not path.is_file():
+        raise HTTPException(status_code=404, detail="Artifact not found.")
+    return FileResponse(path, filename=path.name)

app.py

@@ -0,0 +1,451 @@
+from __future__ import annotations
+
+import io
+import os
+from pathlib import Path
+
+import pandas as pd
+import plotly.express as px
+import streamlit as st
+
+from datapilot.analyst import dataframe_csv, gemini_dataset_summary, inspect_dataset
+from datapilot.config import get_settings
+from datapilot.data import SAMPLE_DATASETS, load_sample
+from datapilot.workflow import run_analysis
+
+st.set_page_config(
+    page_title="DataPilot · Autonomous Data Analyst",
+    page_icon="✦",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
+
+st.markdown(
+    """
+<style>
+@import url('https://fonts.googleapis.com/css2?family=DM+Sans:wght@400;500;600;700&family=Manrope:wght@600;700;800&display=swap');
+:root{--navy:#07111f;--panel:#0e1b2c;--line:#203149;--cyan:#49d7c5;--blue:#6d8dff;--text:#edf4ff;--muted:#91a1b7}
+.stApp{background:radial-gradient(circle at 75% -10%,#17355c 0,transparent 35%),#07111f;color:var(--text)}
+html,body,[class*="css"]{font-family:"DM Sans",sans-serif}
+h1,h2,h3{font-family:"Manrope",sans-serif;letter-spacing:-.03em}
+header[data-testid="stHeader"]{background:transparent}
+div[data-testid="stSidebar"]{background:#091522;border-right:1px solid var(--line)}
+.block-container{max-width:1480px;padding-top:1.1rem;padding-bottom:4rem}
+.brand{display:flex;gap:.75rem;align-items:center;font:800 1.2rem Manrope;color:white;margin:.2rem 0 1.3rem}
+.brand-mark{display:grid;place-items:center;width:34px;height:34px;border-radius:10px;background:linear-gradient(135deg,var(--cyan),var(--blue));color:#07111f}
+.hero{border:1px solid #29405d;background:linear-gradient(125deg,rgba(17,35,57,.96),rgba(9,22,38,.88));border-radius:24px;padding:2rem 2.2rem;margin-bottom:1rem;overflow:hidden;position:relative}
+.hero:after{content:"";position:absolute;width:340px;height:340px;border-radius:50%;right:-100px;top:-190px;background:rgba(73,215,197,.10)}
+.eyebrow{color:var(--cyan);font-size:.73rem;font-weight:700;letter-spacing:.18em;text-transform:uppercase}
+.hero h1{font-size:clamp(2.1rem,4vw,4rem);line-height:1.02;margin:.45rem 0 .7rem;color:white}
+.hero p{max-width:790px;color:#aebdd0;font-size:1.02rem;line-height:1.65;margin:0}
+.stepbar{display:flex;gap:.5rem;flex-wrap:wrap;margin-top:1.4rem}.step{border:1px solid #2d4664;border-radius:999px;padding:.4rem .72rem;color:#9eafc4;font-size:.75rem}.step.on{color:#07111f;background:var(--cyan);border-color:var(--cyan);font-weight:700}
+.panel{background:rgba(14,27,44,.92);border:1px solid var(--line);border-radius:18px;padding:1.15rem 1.25rem;height:100%}
+.kicker{color:var(--cyan);font-size:.72rem;font-weight:700;text-transform:uppercase;letter-spacing:.12em}.muted{color:var(--muted);font-size:.87rem;line-height:1.55}
+.signature{background:linear-gradient(145deg,#11263b,#0b1828);border:1px solid #29435f;border-radius:17px;padding:1rem;margin-top:1rem}.signature strong{color:white}.signature a{color:var(--cyan);text-decoration:none;font-size:.83rem}
+div[data-testid="stMetric"]{background:#0d1b2c;border:1px solid var(--line);padding:15px 17px;border-radius:15px}div[data-testid="stMetric"] label{color:#91a1b7}div[data-testid="stMetricValue"]{color:white}
+.stButton>button,.stDownloadButton>button{border:0;border-radius:11px;background:linear-gradient(135deg,#49d7c5,#6d8dff);color:#07111f;font-weight:800}
+.stButton>button:hover,.stDownloadButton>button:hover{color:#07111f;filter:brightness(1.08)}
+div[data-testid="stFileUploaderDropzone"]{background:#0c1a2b;border:1.5px dashed #3b617c;border-radius:16px;padding:1.3rem}
+div[data-baseweb="tab-list"]{gap:.3rem;background:#0b1828;border:1px solid var(--line);border-radius:13px;padding:.3rem}
+button[data-baseweb="tab"]{border-radius:9px;color:#9caec3}button[data-baseweb="tab"][aria-selected="true"]{background:#172b41;color:white}
+.stDataFrame{border:1px solid var(--line);border-radius:13px;overflow:hidden}
+[data-testid="stAlert"]{border-radius:13px}
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+settings = get_settings()
+for key, default in {
+    "frame": None,
+    "dataset_name": "",
+    "profile": None,
+    "result": None,
+    "ai_summary": "",
+    "chat": [],
+    "target": None,
+}.items():
+    if key not in st.session_state:
+        st.session_state[key] = default
+
+
+def read_upload(uploaded) -> pd.DataFrame:
+    suffix = Path(uploaded.name).suffix.lower()
+    raw = uploaded.getvalue()
+    if len(raw) > settings.max_upload_mb * 1_048_576:
+        raise ValueError(f"File exceeds the {settings.max_upload_mb} MB limit.")
+    stream = io.BytesIO(raw)
+    if suffix in {".csv", ".tsv", ".txt"}:
+        return pd.read_csv(stream, sep="\t" if suffix == ".tsv" else None, engine="python")
+    if suffix in {".xlsx", ".xls"}:
+        return pd.read_excel(stream)
+    if suffix == ".parquet":
+        return pd.read_parquet(stream)
+    if suffix == ".json":
+        try:
+            return pd.read_json(stream)
+        except ValueError:
+            stream.seek(0)
+            return pd.read_json(stream, lines=True)
+    raise ValueError("Use CSV, TSV, Excel, JSON, or Parquet.")
+
+
+with st.sidebar:
+    st.markdown(
+        '<div class="brand"><span class="brand-mark">✦</span>DataPilot</div>',
+        unsafe_allow_html=True,
+    )
+    st.caption("AUTONOMOUS ANALYSIS WORKSPACE")
+    st.markdown("##### Gemini intelligence")
+    default_key = os.getenv("GEMINI_API_KEY", os.getenv("GOOGLE_API_KEY", ""))
+    api_key = st.text_input(
+        "Gemini API key",
+        value=default_key,
+        type="password",
+        help="Masked in the interface and not intentionally written to files or logs.",
+    )
+    model = st.selectbox("Model", ["gemini-2.5-flash", "gemini-2.5-pro", "gemini-2.0-flash"])
+    st.caption("● AI ready" if api_key else "○ Local analysis mode")
+    st.divider()
+    st.markdown("##### Privacy controls")
+    metadata_only = st.toggle(
+        "Metadata-first AI",
+        value=True,
+        help="Send schema, aggregate statistics, and three redacted examples—not the full dataset.",
+    )
+    excluded = st.multiselect(
+        "Exclude columns from AI",
+        list(st.session_state.frame.columns) if st.session_state.frame is not None else [],
+    )
+    st.divider()
+    if st.button("Reset workspace", width="stretch"):
+        for key in ("frame", "profile", "result", "ai_summary", "chat", "target"):
+            st.session_state[key] = (
+                None
+                if key in {"frame", "profile", "result", "target"}
+                else ([] if key == "chat" else "")
+            )
+        st.rerun()
+    st.markdown(
+        """
+    <div class="signature">
+      <div class="kicker">Built & designed by</div>
+      <strong>Dinesh Barri</strong><br>
+      <span class="muted">AI Engineer · Data Scientist</span><br><br>
+      <a href="https://github.com/dineshbarri">GitHub ↗</a>&nbsp;&nbsp;
+      <a href="https://www.linkedin.com/in/dinesh-barri-7654b010b">LinkedIn ↗</a>
+    </div>""",
+        unsafe_allow_html=True,
+    )
+
+loaded = st.session_state.frame is not None
+st.markdown(
+    f"""
+<section class="hero">
+  <div class="eyebrow">Evidence-first autonomous data science</div>
+  <h1>Your data. Explained.<br>Decisions, accelerated.</h1>
+  <p>Upload a dataset and DataPilot immediately inspects its structure, surfaces quality risks,
+  recommends analytical targets, creates interactive evidence, and prepares a leakage-safe
+  machine-learning study—with Gemini available for grounded interpretation.</p>
+  <div class="stepbar">
+    <span class="step {"on" if loaded else ""}">01 · Connect</span>
+    <span class="step {"on" if loaded else ""}">02 · Inspect</span>
+    <span class="step {"on" if st.session_state.ai_summary else ""}">03 · Interpret</span>
+    <span class="step {"on" if st.session_state.result else ""}">04 · Model</span>
+    <span class="step {"on" if st.session_state.result else ""}">05 · Deliver</span>
+  </div>
+</section>""",
+    unsafe_allow_html=True,
+)
+
+if not loaded:
+    left, right = st.columns([1.35, 0.65], gap="large")
+    with left:
+        st.markdown('<div class="kicker">Start a new analysis</div>', unsafe_allow_html=True)
+        st.subheader("Drop in your dataset")
+        uploaded = st.file_uploader(
+            "Upload dataset",
+            type=["csv", "tsv", "txt", "xlsx", "xls", "json", "parquet"],
+            label_visibility="collapsed",
+        )
+        st.caption("CSV · TSV · Excel · JSON · Parquet  |  Raw data remains in this session.")
+        if uploaded:
+            try:
+                with st.status("DataPilot is inspecting your dataset…", expanded=True) as status:
+                    st.write("Validating file structure")
+                    frame = read_upload(uploaded)
+                    st.write("Profiling columns, missingness, cardinality, and target candidates")
+                    profile = inspect_dataset(frame)
+                    st.session_state.frame = frame
+                    st.session_state.profile = profile
+                    st.session_state.dataset_name = uploaded.name
+                    status.update(label="Dataset ready", state="complete")
+                st.rerun()
+            except Exception as exc:
+                st.error(f"Upload could not be processed: {exc}")
+    with right:
+        st.markdown(
+            '<div class="panel"><div class="kicker">Try it instantly</div><h3>Explore a trusted demo</h3><p class="muted">Load a complete classification or regression dataset and see the full analyst workflow.</p></div>',
+            unsafe_allow_html=True,
+        )
+        demo = st.selectbox("Demo dataset", list(SAMPLE_DATASETS))
+        if st.button("Load demo workspace", width="stretch"):
+            frame, target, name = load_sample(SAMPLE_DATASETS[demo])
+            st.session_state.frame, st.session_state.target = frame, target
+            st.session_state.dataset_name = name
+            st.session_state.profile = inspect_dataset(frame)
+            st.rerun()
+    st.stop()
+
+frame: pd.DataFrame = st.session_state.frame
+profile = st.session_state.profile or inspect_dataset(frame)
+brief = profile["brief"]
+
+metrics = st.columns(6)
+metrics[0].metric("Rows", f"{brief.rows:,}")
+metrics[1].metric("Columns", f"{brief.columns:,}")
+metrics[2].metric("Numeric", brief.numeric)
+metrics[3].metric("Categorical", brief.categorical)
+metrics[4].metric("Missing cells", f"{brief.missing_cells:,}")
+metrics[5].metric("Quality score", f"{profile['quality_score']}/100")
+
+overview, quality, explore, ai_tab, model_tab, deliver = st.tabs(
+    ["Overview", "Data quality", "Explore", "AI insights", "Model lab", "Deliver"]
+)
+
+with overview:
+    st.subheader(st.session_state.dataset_name)
+    st.caption(f"Dataset fingerprint {brief.fingerprint} · {brief.memory_mb:.2f} MB in memory")
+    first, last, sample = st.tabs(["First 5 rows", "Last 5 rows", "Random sample"])
+    first.dataframe(frame.head(), width="stretch", hide_index=True)
+    last.dataframe(frame.tail(), width="stretch", hide_index=True)
+    sample.dataframe(
+        frame.sample(min(5, len(frame)), random_state=42), width="stretch", hide_index=True
+    )
+    st.markdown("#### Data dictionary")
+    st.dataframe(
+        profile["dictionary"].drop(columns=["issue_count"]), width="stretch", hide_index=True
+    )
+
+with quality:
+    a, b = st.columns([0.75, 1.25])
+    with a:
+        st.markdown("#### Quality signals")
+        st.metric("Duplicate rows", f"{brief.duplicate_rows:,}")
+        st.metric("Completeness", f"{100 - brief.missing_cells / max(1, frame.size) * 100:.1f}%")
+        flagged = profile["dictionary"].query("issue_count > 0")
+        st.metric("Flagged columns", len(flagged))
+        st.info("DataPilot reports evidence first. No rows or values are changed without approval.")
+    with b:
+        missing = profile["missing"][profile["missing"] > 0].sort_values()
+        if len(missing):
+            fig = px.bar(
+                x=missing.values,
+                y=missing.index,
+                orientation="h",
+                labels={"x": "Missing values", "y": "Column"},
+                title="Missing values by column",
+                color=missing.values,
+                color_continuous_scale=["#49d7c5", "#6d8dff"],
+            )
+            fig.update_layout(
+                template="plotly_dark",
+                paper_bgcolor="#0e1b2c",
+                plot_bgcolor="#0e1b2c",
+                coloraxis_showscale=False,
+            )
+            st.plotly_chart(fig, width="stretch")
+        else:
+            st.success("No missing values detected.")
+        if len(flagged):
+            st.dataframe(flagged.drop(columns=["issue_count"]), width="stretch", hide_index=True)
+
+with explore:
+    numeric = profile["numeric"]
+    if numeric:
+        selected = st.selectbox("Explore a numerical feature", numeric)
+        c1, c2 = st.columns(2)
+        fig = px.histogram(
+            frame,
+            x=selected,
+            marginal="box",
+            title=f"Distribution of {selected}",
+            color_discrete_sequence=["#49d7c5"],
+        )
+        fig.update_layout(template="plotly_dark", paper_bgcolor="#0e1b2c", plot_bgcolor="#0e1b2c")
+        c1.plotly_chart(fig, width="stretch")
+        if not profile["correlation"].empty:
+            heat = px.imshow(
+                profile["correlation"],
+                text_auto=".2f",
+                aspect="auto",
+                color_continuous_scale=["#1a2940", "#49d7c5", "#f4b860"],
+                title="Numeric correlation map",
+            )
+            heat.update_layout(template="plotly_dark", paper_bgcolor="#0e1b2c")
+            c2.plotly_chart(heat, width="stretch")
+        else:
+            c2.info("Add another numerical column to calculate correlations.")
+        st.dataframe(frame[numeric].describe().T, width="stretch")
+    else:
+        st.info("This dataset has no numerical columns. Use the categorical overview below.")
+    categories = profile["categorical"]
+    if categories:
+        selected_cat = st.selectbox("Explore a categorical feature", categories)
+        counts = frame[selected_cat].astype(str).value_counts().head(20).reset_index()
+        fig = px.bar(
+            counts,
+            x="count",
+            y=selected_cat,
+            orientation="h",
+            title=f"Top values · {selected_cat}",
+            color="count",
+            color_continuous_scale=["#49d7c5", "#6d8dff"],
+        )
+        fig.update_layout(
+            template="plotly_dark",
+            paper_bgcolor="#0e1b2c",
+            plot_bgcolor="#0e1b2c",
+            coloraxis_showscale=False,
+        )
+        st.plotly_chart(fig, width="stretch")
+
+with ai_tab:
+    st.markdown("#### Ask Gemini to interpret the computed evidence")
+    st.caption(
+        "AI interpretation based on dataset metadata and limited redacted samples. Verify against source documentation."
+    )
+    if not api_key:
+        st.warning(
+            "Enter a Gemini API key in the sidebar. Deterministic profiling remains fully available without AI."
+        )
+    if st.button("Generate AI analyst brief", disabled=not bool(api_key)):
+        try:
+            with st.status("Gemini is reviewing the evidence package…", expanded=True) as status:
+                st.write("Redacting potential PII and excluded columns")
+                st.write("Sending schema, aggregate statistics, and three examples")
+                summary = gemini_dataset_summary(frame, profile, api_key, model, excluded)
+                st.session_state.ai_summary = summary
+                status.update(label="AI analyst brief ready", state="complete")
+        except ValueError as exc:
+            st.error(str(exc))
+    if st.session_state.ai_summary:
+        st.markdown(st.session_state.ai_summary)
+    with st.expander("What may be sent to Gemini"):
+        st.write(
+            "Column metadata, aggregate statistics, target candidates, quality score, and up to three redacted example rows."
+        )
+        st.write(
+            "Automatically excluded potential PII:",
+            [
+                c
+                for c in frame.columns
+                if any(
+                    k in str(c).lower() for k in ("email", "phone", "address", "name", "account")
+                )
+            ]
+            or "None detected",
+        )
+
+with model_tab:
+    st.markdown("#### Confirm the analytical target")
+    candidates = pd.DataFrame(profile["targets"])
+    st.dataframe(candidates, width="stretch", hide_index=True)
+    default_target = st.session_state.target or (
+        profile["targets"][0]["column"] if profile["targets"] else frame.columns[-1]
+    )
+    target = st.selectbox(
+        "Target column", list(frame.columns), index=list(frame.columns).index(default_target)
+    )
+    st.caption("DataPilot will not train supervised models until you confirm this selection.")
+    if frame[target].nunique(dropna=True) < 2:
+        st.error("The selected target has fewer than two observed values.")
+    run = st.button(
+        "Run autonomous model study",
+        type="primary",
+        disabled=frame[target].nunique(dropna=True) < 2,
+    )
+    if run:
+        try:
+            progress = st.progress(0, text="Preparing agent graph")
+            progress.progress(12, text="Data Quality Agent · auditing risks")
+            with st.spinner(
+                "LangGraph agents are profiling, planning, training, evaluating, and explaining…"
+            ):
+                result = run_analysis(frame, target, st.session_state.dataset_name, settings)
+            progress.progress(100, text="Analysis complete")
+            st.session_state.result = result.model_dump(mode="json")
+            st.success(
+                "Model study completed with leakage-safe preprocessing and cross-validation."
+            )
+        except Exception as exc:
+            st.error(f"Model study failed: {exc}")
+    result = st.session_state.result
+    if result:
+        best = result["model_results"][0]
+        c1, c2, c3 = st.columns(3)
+        c1.metric("Selected model", result["best_model"])
+        c2.metric(
+            "One-time test " + best["primary_metric"].replace("_", " ").title(),
+            f"{best['final_test_score']:.3f}",
+        )
+        c3.metric("CV mean", f"{best['cross_validation_mean']:.3f}")
+        results = pd.DataFrame(result["model_results"])
+        fig = px.bar(
+            results.sort_values("selection_score"),
+            x="selection_score",
+            y="name",
+            orientation="h",
+            color="selection_score",
+            title="Training-CV model selection",
+            color_continuous_scale=["#344b69", "#49d7c5"],
+        )
+        fig.update_layout(
+            template="plotly_dark",
+            paper_bgcolor="#0e1b2c",
+            plot_bgcolor="#0e1b2c",
+            coloraxis_showscale=False,
+        )
+        st.plotly_chart(fig, width="stretch")
+        st.dataframe(results, width="stretch", hide_index=True)
+        st.markdown("#### Agent execution trace")
+        st.dataframe(pd.DataFrame(result["trace"]), width="stretch", hide_index=True)
+
+with deliver:
+    st.markdown("#### Export your evidence")
+    c1, c2 = st.columns(2)
+    c1.download_button(
+        "Download original dataset · CSV",
+        dataframe_csv(frame),
+        file_name=f"{Path(st.session_state.dataset_name).stem}_datapilot.csv",
+        mime="text/csv",
+        width="stretch",
+    )
+    c2.download_button(
+        "Download data dictionary · CSV",
+        dataframe_csv(profile["dictionary"].drop(columns=["issue_count"])),
+        file_name="datapilot_data_dictionary.csv",
+        mime="text/csv",
+        width="stretch",
+    )
+    result = st.session_state.result
+    if result:
+        st.markdown("#### Model and report artifacts")
+        columns = st.columns(min(4, len(result["artifacts"])))
+        for column, (name, raw_path) in zip(columns, result["artifacts"].items(), strict=False):
+            path = Path(raw_path)
+            if path.exists():
+                column.download_button(
+                    name.replace("_", " ").title(),
+                    path.read_bytes(),
+                    file_name=path.name,
+                    width="stretch",
+                )
+    else:
+        st.info(
+            "Run a model study to unlock the fitted pipeline, model card, metrics, and HTML report."
+        )
+
+st.caption(
+    "DataPilot provides exploratory decision support. Predictive associations do not establish causality."
+)

artifacts/run_f13555d6077f/MODEL_CARD.md

@@ -0,0 +1,41 @@
+# Model Card — iris
+
+## Model details
+
+- Run ID: `run_f13555d6077f`
+- Task: classification
+- Target: `target`
+- Selected model: **Logistic Regression**
+- Training-CV selection metric: `balanced_accuracy = 0.9583`
+- One-time untouched test metric: `balanced_accuracy = 0.9333`
+- Training rows before split: 150
+
+## Intended use
+
+Exploratory decision support and portfolio demonstration. Validate with domain-specific,
+out-of-time data before any consequential or production use.
+
+## Evaluation
+
+```json
+{
+  "accuracy": 0.9333,
+  "balanced_accuracy": 0.9333,
+  "f1_weighted": 0.9333
+}
+```
+
+## Data-quality observations
+
+- 1 exact duplicate rows can bias validation.
+
+## Explainability
+
+Method: **Permutation importance**. Importance values are predictive associations,
+not evidence of causation.
+
+## Limitations
+
+- Results depend on the uploaded dataset and chosen target.
+- Automated task inference can be wrong; a domain owner should confirm the objective.
+- Fairness, privacy, and legal review are outside the automatic approval gate.

artifacts/run_f13555d6077f/analysis_report.html

@@ -0,0 +1,23 @@
+<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
+<title>DataPilot AI report</title>
+<style>
+body{font-family:Inter,system-ui,sans-serif;max-width:1000px;margin:40px auto;padding:0 24px;color:#172033}
+h1{color:#5537d8} .hero{background:#f4f1ff;border:1px solid #d9d0ff;padding:24px;border-radius:18px}
+.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:16px;margin:20px 0}
+.card{border:1px solid #e2e6ef;border-radius:14px;padding:18px} table{border-collapse:collapse;width:100%}
+td,th{border-bottom:1px solid #e2e6ef;padding:10px;text-align:left} small{color:#667085}
+</style></head><body>
+<div class="hero"><h1>DataPilot AI Analysis Report</h1>
+<p>iris · Run run_f13555d6077f</p></div>
+<div class="grid">
+<div class="card"><small>Selected model</small><h2>Logistic Regression</h2></div>
+<div class="card"><small>Test balanced_accuracy</small><h2>0.933</h2></div>
+<div class="card"><small>Rows analyzed</small><h2>150</h2></div>
+</div>
+<h2>Executive findings</h2><ul><li>The analysis used 150 rows and 5 columns for a classification task targeting &#x27;target&#x27;.</li><li>Logistic Regression ranked first with training CV balanced_accuracy 0.958; its one-time test score was 0.933.</li><li>1 data-quality observations were recorded; 0 are critical.</li><li>The strongest predictive signals were petal_width_(cm), petal_length_(cm), sepal_length_(cm) according to permutation importance.</li></ul>
+<h2>Evaluation</h2><table><tr><th>Metric</th><th>Value</th></tr><tr><td>accuracy</td><td>0.9333</td></tr><tr><td>balanced_accuracy</td><td>0.9333</td></tr><tr><td>f1_weighted</td><td>0.9333</td></tr></table>
+<h2>Data quality</h2><table><tr><th>Severity</th><th>Code</th><th>Observation</th></tr><tr><td>warning</td><td>DUPLICATE_ROWS</td><td>1 exact duplicate rows can bias validation.</td></tr></table>
+<h2>Recommendations</h2><ol><li>Validate performance on fresh, out-of-time data before production deployment.</li><li>Review suspected leakage and identifier columns with a domain owner.</li><li>Monitor input drift and the primary metric after deployment.</li></ol>
+<p><small>Generated from computed evidence. Predictive findings do not establish causality.</small></p>
+</body></html>

artifacts/run_f13555d6077f/metrics.json

@@ -0,0 +1,228 @@
+{
+  "run_id": "run_f13555d6077f",
+  "status": "completed",
+  "dataset_name": "iris",
+  "profile": {
+    "rows": 150,
+    "columns": 5,
+    "numeric_columns": [
+      "sepal_length_(cm)",
+      "sepal_width_(cm)",
+      "petal_length_(cm)",
+      "petal_width_(cm)",
+      "target"
+    ],
+    "categorical_columns": [],
+    "datetime_columns": [],
+    "duplicate_rows": 1,
+    "missing_cells": 0,
+    "missing_rate": 0.0,
+    "memory_mb": 0.006,
+    "target": "target",
+    "task_type": "classification",
+    "target_cardinality": 3
+  },
+  "plan": {
+    "objective": "Predict 'target' and produce reproducible, evidence-backed insights.",
+    "target": "target",
+    "task_type": "classification",
+    "primary_metric": "balanced_accuracy",
+    "validation_strategy": "Training-only stratified cross-validation; untouched final test evaluation",
+    "candidate_models": [
+      "Logistic Regression",
+      "Random Forest",
+      "Extra Trees",
+      "Histogram Gradient Boosting",
+      "XGBoost (when installed)"
+    ],
+    "risk_controls": [
+      "Drop rows with missing target before split",
+      "Fit imputers, encoders, and scalers on training folds only",
+      "Flag leakage-like names and identifier cardinality",
+      "Require critic quality gate before explanation"
+    ]
+  },
+  "quality_issues": [
+    {
+      "code": "DUPLICATE_ROWS",
+      "severity": "warning",
+      "column": null,
+      "message": "1 exact duplicate rows can bias validation.",
+      "evidence_ids": [
+        "EV-9000641D"
+      ]
+    }
+  ],
+  "evidence": [
+    {
+      "evidence_id": "EV-84BCE8AD",
+      "claim": "Dataset missingness was measured across all cells.",
+      "metric": "missing_rate",
+      "value": 0.0,
+      "source": "uploaded_dataset",
+      "method": "pandas.isna"
+    },
+    {
+      "evidence_id": "EV-9000641D",
+      "claim": "Exact duplicate rows were counted before splitting.",
+      "metric": "duplicate_rows",
+      "value": 1,
+      "source": "uploaded_dataset",
+      "method": "pandas.duplicated"
+    },
+    {
+      "evidence_id": "EV-A9BE42C5",
+      "claim": "Rows with missing labels cannot be used for supervised training.",
+      "metric": "missing_target_rows",
+      "value": 0,
+      "source": "column:target",
+      "method": "pandas.isna"
+    },
+    {
+      "evidence_id": "EV-76E9ED71",
+      "claim": "Class imbalance was measured using the minority-class share.",
+      "metric": "minority_class_share",
+      "value": 0.3333,
+      "source": "column:target",
+      "method": "normalized value counts"
+    }
+  ],
+  "model_results": [
+    {
+      "name": "Logistic Regression",
+      "primary_metric": "balanced_accuracy",
+      "primary_score": 0.9583333333333334,
+      "metrics": {
+        "accuracy": 0.9333,
+        "balanced_accuracy": 0.9333,
+        "f1_weighted": 0.9333
+      },
+      "cross_validation_mean": 0.9583333333333334,
+      "cross_validation_std": 0.026352313834736508,
+      "training_seconds": 0.228,
+      "selection_score": 0.9583333333333334,
+      "final_test_score": 0.9333,
+      "final_test_metrics": {
+        "accuracy": 0.9333,
+        "balanced_accuracy": 0.9333,
+        "f1_weighted": 0.9333
+      }
+    },
+    {
+      "name": "Extra Trees",
+      "primary_metric": "balanced_accuracy",
+      "primary_score": 0.9583333333333334,
+      "metrics": {},
+      "cross_validation_mean": 0.9583333333333334,
+      "cross_validation_std": 0.026352313834736508,
+      "training_seconds": 2.901,
+      "selection_score": 0.9583333333333334,
+      "final_test_score": null,
+      "final_test_metrics": {}
+    },
+    {
+      "name": "Random Forest",
+      "primary_metric": "balanced_accuracy",
+      "primary_score": 0.95,
+      "metrics": {},
+      "cross_validation_mean": 0.95,
+      "cross_validation_std": 0.0311804782231162,
+      "training_seconds": 2.641,
+      "selection_score": 0.95,
+      "final_test_score": null,
+      "final_test_metrics": {}
+    }
+  ],
+  "model_failures": [],
+  "best_model": "Logistic Regression",
+  "critic": {
+    "approved": true,
+    "score": 0.9583333333333334,
+    "threshold": 0.55,
+    "reasons": [
+      "Performance and validation consistency passed the configured quality gate."
+    ],
+    "retry_number": 0
+  },
+  "explainability": {
+    "method": "Permutation importance",
+    "feature_importance": {
+      "petal_width_(cm)": 0.213333,
+      "petal_length_(cm)": 0.2,
+      "sepal_length_(cm)": 0.06,
+      "sepal_width_(cm)": 0.046667
+    },
+    "caveats": [
+      "Permutation importance can dilute importance among correlated features.",
+      "Feature importance is predictive, not causal."
+    ]
+  },
+  "executive_summary": [
+    "The analysis used 150 rows and 5 columns for a classification task targeting 'target'.",
+    "Logistic Regression ranked first with training CV balanced_accuracy 0.958; its one-time test score was 0.933.",
+    "1 data-quality observations were recorded; 0 are critical.",
+    "The strongest predictive signals were petal_width_(cm), petal_length_(cm), sepal_length_(cm) according to permutation importance."
+  ],
+  "recommendations": [
+    "Validate performance on fresh, out-of-time data before production deployment.",
+    "Review suspected leakage and identifier columns with a domain owner.",
+    "Monitor input drift and the primary metric after deployment."
+  ],
+  "artifacts": {},
+  "trace": [
+    {
+      "agent": "Data Quality Agent",
+      "status": "completed",
+      "duration_seconds": 0.005,
+      "detail": "Recorded 1 quality observations."
+    },
+    {
+      "agent": "EDA Agent",
+      "status": "completed",
+      "duration_seconds": 0.058,
+      "detail": "Computed DuckDB-backed dataset overview."
+    },
+    {
+      "agent": "Statistical Analysis Agent",
+      "status": "completed",
+      "duration_seconds": 0.001,
+      "detail": "Measured target distribution and associations."
+    },
+    {
+      "agent": "Planning Agent",
+      "status": "completed",
+      "duration_seconds": 0.0,
+      "detail": "Selected balanced_accuracy as primary metric."
+    },
+    {
+      "agent": "Feature Engineering Agent",
+      "status": "completed",
+      "duration_seconds": 0.0,
+      "detail": "Created leakage-safe ColumnTransformer plan."
+    },
+    {
+      "agent": "Modeling Agent",
+      "status": "completed",
+      "duration_seconds": 5.916,
+      "detail": "Compared 3 models; Logistic Regression ranked first."
+    },
+    {
+      "agent": "Evaluation / Critic Agent",
+      "status": "completed",
+      "duration_seconds": 0.0,
+      "detail": "Approved analysis."
+    },
+    {
+      "agent": "Explainability Agent",
+      "status": "completed",
+      "duration_seconds": 0.331,
+      "detail": "Generated Permutation importance explanations."
+    },
+    {
+      "agent": "Executive Insights Agent",
+      "status": "completed",
+      "duration_seconds": 0.0,
+      "detail": "Created evidence-grounded narrative with deterministic metric provenance."
+    }
+  ]
+}

artifacts/run_f13555d6077f/reproduction.json

@@ -0,0 +1,8 @@
+{
+  "run_id": "run_f13555d6077f",
+  "random_state": 42,
+  "test_size": 0.2,
+  "target": "target",
+  "task_type": "classification",
+  "best_model": "Logistic Regression"
+}

datapilot/__init__.py

@@ -0,0 +1,3 @@
+"""DataPilot AI: evidence-grounded autonomous data science."""
+
+__version__ = "1.0.0"

datapilot/analyst.py

@@ -0,0 +1,200 @@
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+TARGET_WORDS = {
+    "target": 1.0,
+    "label": 1.0,
+    "outcome": 0.95,
+    "class": 0.9,
+    "churn": 0.95,
+    "fraud": 0.95,
+    "default": 0.9,
+    "price": 0.8,
+    "revenue": 0.75,
+    "sales": 0.75,
+    "diagnosis": 0.9,
+    "status": 0.7,
+    "response": 0.8,
+    "converted": 0.9,
+}
+PII_PATTERN = re.compile(r"(email|phone|mobile|address|ssn|passport|account|name)", re.I)
+
+
+@dataclass
+class DatasetBrief:
+    fingerprint: str
+    rows: int
+    columns: int
+    numeric: int
+    categorical: int
+    datetime: int
+    missing_cells: int
+    duplicate_rows: int
+    memory_mb: float
+
+
+def inspect_dataset(frame: pd.DataFrame) -> dict[str, Any]:
+    """Compute an immediate analyst profile without requiring a target."""
+    numeric = list(frame.select_dtypes(include=np.number).columns)
+    datetime = list(frame.select_dtypes(include=["datetime", "datetimetz"]).columns)
+    categorical = [c for c in frame.columns if c not in numeric and c not in datetime]
+    missing = frame.isna().sum()
+    brief = DatasetBrief(
+        fingerprint=hashlib.sha256(
+            pd.util.hash_pandas_object(frame, index=True).values.tobytes()
+        ).hexdigest()[:12],
+        rows=len(frame),
+        columns=len(frame.columns),
+        numeric=len(numeric),
+        categorical=len(categorical),
+        datetime=len(datetime),
+        missing_cells=int(missing.sum()),
+        duplicate_rows=int(frame.duplicated().sum()),
+        memory_mb=round(float(frame.memory_usage(deep=True).sum() / 1_048_576), 2),
+    )
+    dictionary = build_data_dictionary(frame)
+    quality_score = max(
+        0,
+        round(
+            100
+            - (brief.missing_cells / max(1, frame.size)) * 45
+            - (brief.duplicate_rows / max(1, brief.rows)) * 25
+            - sum(dictionary["issue_count"].clip(upper=3)) / max(1, len(dictionary)) * 4
+        ),
+    )
+    return {
+        "brief": brief,
+        "dictionary": dictionary,
+        "targets": rank_target_candidates(frame),
+        "quality_score": quality_score,
+        "missing": missing.sort_values(ascending=False),
+        "numeric": numeric,
+        "categorical": categorical,
+        "datetime": datetime,
+        "correlation": frame[numeric].corr(numeric_only=True)
+        if len(numeric) > 1
+        else pd.DataFrame(),
+    }
+
+
+def build_data_dictionary(frame: pd.DataFrame) -> pd.DataFrame:
+    """Create an evidence-based data dictionary for every column."""
+    rows: list[dict[str, Any]] = []
+    for column in frame.columns:
+        series = frame[column]
+        unique, missing = int(series.nunique(dropna=True)), int(series.isna().sum())
+        issues: list[str] = []
+        if missing:
+            issues.append("Missing values")
+        if unique <= 1:
+            issues.append("Constant")
+        if len(frame) and unique / len(frame) > 0.98:
+            issues.append("Identifier-like")
+        if PII_PATTERN.search(str(column)):
+            issues.append("Potential PII")
+        role = "Numeric feature" if pd.api.types.is_numeric_dtype(series) else "Categorical feature"
+        if pd.api.types.is_datetime64_any_dtype(series):
+            role = "Datetime"
+        elif unique == len(frame) and len(frame) > 10:
+            role = "Identifier"
+        rows.append(
+            {
+                "column": str(column),
+                "type": str(series.dtype),
+                "role": role,
+                "unique": unique,
+                "missing": missing,
+                "missing_%": round(missing / max(1, len(frame)) * 100, 2),
+                "example_values": ", ".join(map(str, series.dropna().astype(str).unique()[:3]))[
+                    :90
+                ],
+                "issues": ", ".join(issues) or "None detected",
+                "issue_count": len(issues),
+            }
+        )
+    return pd.DataFrame(rows)
+
+
+def rank_target_candidates(frame: pd.DataFrame) -> list[dict[str, Any]]:
+    """Rank targets while explicitly leaving confirmation to the user."""
+    ranked: list[dict[str, Any]] = []
+    for position, column in enumerate(frame.columns):
+        series, name = frame[column], str(column).lower().strip()
+        cardinality = int(series.nunique(dropna=True))
+        score = max((v for word, v in TARGET_WORDS.items() if word in name), default=0.0)
+        if 2 <= cardinality <= max(20, int(len(frame) * 0.1)):
+            score += 0.22
+        if position == len(frame.columns) - 1:
+            score += 0.12
+        if cardinality >= max(10, int(len(frame) * 0.95)):
+            score -= 0.45
+        if series.isna().mean() > 0.5 or cardinality < 2:
+            score -= 0.5
+        task = "classification" if cardinality <= max(20, int(len(frame) * 0.05)) else "regression"
+        if pd.api.types.is_numeric_dtype(series) and cardinality > 20:
+            task = "regression"
+        ranked.append(
+            {
+                "column": str(column),
+                "task": task,
+                "confidence": round(max(0, min(score, 0.99)), 2),
+                "reason": f"{cardinality:,} distinct values; "
+                + ("name/position signals detected" if score > 0.3 else "weak heuristic evidence"),
+            }
+        )
+    return sorted(ranked, key=lambda item: item["confidence"], reverse=True)[:5]
+
+
+def ai_context(frame: pd.DataFrame, profile: dict[str, Any], excluded: list[str]) -> dict[str, Any]:
+    """Build a bounded, redacted payload suitable for AI interpretation."""
+    safe = frame.drop(columns=[c for c in excluded if c in frame], errors="ignore").copy()
+    pii = [c for c in safe.columns if PII_PATTERN.search(str(c))]
+    safe = safe.drop(columns=pii, errors="ignore")
+    return {
+        "shape": list(frame.shape),
+        "columns": profile["dictionary"].drop(columns=["issue_count"]).to_dict(orient="records"),
+        "numeric_summary": safe.select_dtypes(include=np.number).describe().round(3).to_dict(),
+        "sample": safe.head(3).replace({np.nan: None}).to_dict(orient="records"),
+        "excluded_columns": sorted(set(excluded + pii)),
+        "quality_score": profile["quality_score"],
+        "target_candidates": profile["targets"],
+    }
+
+
+def gemini_dataset_summary(
+    frame: pd.DataFrame, profile: dict[str, Any], api_key: str, model: str, excluded: list[str]
+) -> str:
+    """Ask Gemini for an evidence-bounded analyst narrative."""
+    if not api_key:
+        raise ValueError("Enter a Gemini API key to generate AI interpretation.")
+    try:
+        from google import genai
+    except ImportError as exc:
+        raise ValueError("Install the AI extra: pip install google-genai") from exc
+    prompt = """You are DataPilot, a rigorous senior data analyst. Use ONLY the supplied JSON.
+Return concise Markdown with exactly these headings: Finding, Evidence, Interpretation,
+Limitation, Recommendation. Explain likely row grain and useful business questions, but label
+uncertain semantics as assumptions. Never invent values, origin, or causal claims.
+DATA:\n""" + json.dumps(ai_context(frame, profile, excluded), default=str)
+    try:
+        response = genai.Client(api_key=api_key).models.generate_content(
+            model=model, contents=prompt
+        )
+        return response.text
+    except Exception as exc:
+        raise ValueError(f"Gemini request failed: {str(exc)[:240]}") from exc
+
+
+def dataframe_csv(frame: pd.DataFrame) -> bytes:
+    buffer = io.StringIO()
+    frame.to_csv(buffer, index=False)
+    return buffer.getvalue().encode("utf-8")

datapilot/config.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_name: str = "DataPilot AI"
+    environment: str = "development"
+    artifact_root: Path = Path("artifacts")
+    database_url: str = "sqlite:///artifacts/datapilot.db"
+    max_upload_mb: int = Field(default=25, ge=1, le=250)
+    max_rows: int = Field(default=100_000, ge=100)
+    max_columns: int = Field(default=250, ge=2)
+    max_categories_per_feature: int = Field(default=100, ge=10, le=10_000)
+    max_encoded_features: int = Field(default=5_000, ge=100, le=100_000)
+    api_key: str | None = None
+    requests_per_minute: int = Field(default=30, ge=1, le=10_000)
+    random_state: int = 42
+    test_size: float = Field(default=0.2, gt=0.05, lt=0.5)
+    max_critic_retries: int = Field(default=1, ge=0, le=3)
+    min_classification_score: float = 0.55
+    min_regression_score: float = 0.15
+    optuna_trials: int = Field(default=8, ge=0, le=50)
+    enable_mlflow: bool = False
+    mlflow_tracking_uri: str = "file:./artifacts/mlruns"
+    gemini_api_key: str | None = None
+    gemini_model: str = "gemini-2.5-flash"
+    cors_origins: str = "http://localhost:8501"
+
+    def ensure_directories(self) -> None:
+        self.artifact_root.mkdir(parents=True, exist_ok=True)
+
+
+@lru_cache
+def get_settings() -> Settings:
+    settings = Settings()
+    settings.ensure_directories()
+    return settings

datapilot/data.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import io
+from pathlib import Path
+from typing import BinaryIO
+
+import duckdb
+import pandas as pd
+from sklearn.datasets import load_breast_cancer, load_diabetes, load_iris
+
+from datapilot.config import Settings
+
+SAMPLE_DATASETS = {
+    "Iris classification": "iris",
+    "Breast cancer classification": "breast_cancer",
+    "Diabetes progression regression": "diabetes",
+}
+
+
+def load_sample(name: str) -> tuple[pd.DataFrame, str, str]:
+    loaders = {
+        "iris": load_iris,
+        "breast_cancer": load_breast_cancer,
+        "diabetes": load_diabetes,
+    }
+    if name not in loaders:
+        raise ValueError(f"Unknown sample dataset: {name}")
+    bundle = loaders[name](as_frame=True)
+    frame = bundle.frame.copy()
+    frame.columns = [str(column).replace(" ", "_") for column in frame.columns]
+    target = str(bundle.target.name).replace(" ", "_")
+    return frame, target, name
+
+
+def read_dataset(
+    source: bytes | BinaryIO | str | Path,
+    filename: str,
+    settings: Settings,
+) -> pd.DataFrame:
+    suffix = Path(filename).suffix.lower()
+    if suffix not in {".csv", ".parquet"}:
+        raise ValueError("Only CSV and Parquet files are supported.")
+    if isinstance(source, bytes):
+        if len(source) > settings.max_upload_mb * 1024 * 1024:
+            raise ValueError(f"File exceeds the {settings.max_upload_mb} MB upload limit.")
+        stream: BinaryIO | str | Path = io.BytesIO(source)
+    else:
+        stream = source
+    frame = pd.read_csv(stream) if suffix == ".csv" else pd.read_parquet(stream)
+    validate_shape(frame, settings)
+    frame.columns = _unique_columns([str(column).strip() for column in frame.columns])
+    return frame
+
+
+def validate_shape(frame: pd.DataFrame, settings: Settings) -> None:
+    if frame.empty:
+        raise ValueError("The dataset is empty.")
+    if len(frame) > settings.max_rows:
+        raise ValueError(f"Dataset has {len(frame):,} rows; limit is {settings.max_rows:,}.")
+    if len(frame.columns) > settings.max_columns:
+        raise ValueError(
+            f"Dataset has {len(frame.columns):,} columns; limit is {settings.max_columns:,}."
+        )
+    if len(frame.columns) < 2:
+        raise ValueError("At least one feature and one target column are required.")
+
+
+def _unique_columns(columns: list[str]) -> list[str]:
+    seen: dict[str, int] = {}
+    result: list[str] = []
+    for raw in columns:
+        name = raw or "unnamed"
+        count = seen.get(name, 0)
+        seen[name] = count + 1
+        result.append(name if count == 0 else f"{name}_{count}")
+    return result
+
+
+def duckdb_overview(frame: pd.DataFrame) -> dict[str, object]:
+    connection = duckdb.connect(database=":memory:")
+    try:
+        connection.register("dataset", frame)
+        row = connection.execute(
+            """
+            SELECT
+              COUNT(*) AS row_count
+            FROM dataset
+            """
+        ).fetchone()
+        numeric = frame.select_dtypes(include="number")
+        correlations = (
+            numeric.corr(numeric_only=True).round(4).fillna(0).to_dict()
+            if len(numeric.columns) > 1
+            else {}
+        )
+        return {
+            "row_count": int(row[0]),
+            "duplicate_rows": int(frame.duplicated().sum()),
+            "correlations": correlations,
+        }
+    finally:
+        connection.close()

datapilot/evaluation.py

@@ -0,0 +1,58 @@
+"""Deterministic quality gates for evidence-grounded LLM narratives."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from dataclasses import asdict, dataclass
+
+EVIDENCE_ID = re.compile(r"\bEVD-[A-Z0-9-]+\b")
+NUMBER = re.compile(r"(?<![A-Za-z])[-+]?\d+(?:\.\d+)?%?")
+PII = re.compile(r"[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}|(?:\+?\d[\d .()-]{7,}\d)", re.I)
+INJECTION = re.compile(
+    r"ignore (?:all |the )?(?:previous|prior) instructions|system prompt|developer message",
+    re.I,
+)
+
+
+@dataclass(frozen=True)
+class NarrativeEvaluation:
+    supported_evidence_ids: bool
+    unsupported_numbers: list[str]
+    leaked_pii: bool
+    prompt_injection_echo: bool
+    score: float
+
+    def to_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+def evaluate_narrative(
+    output: str,
+    allowed_evidence_ids: Iterable[str],
+    allowed_numbers: Iterable[str | float | int],
+) -> NarrativeEvaluation:
+    """Score an AI narrative for evidence citations, numeric faithfulness, and safety."""
+    allowed_ids = set(allowed_evidence_ids)
+    cited = set(EVIDENCE_ID.findall(output))
+    supported_ids = cited.issubset(allowed_ids) and bool(cited)
+    allowed_numeric = {_normalize_number(str(value)) for value in allowed_numbers}
+    narrative_without_ids = EVIDENCE_ID.sub("", output)
+    found = {_normalize_number(value) for value in NUMBER.findall(narrative_without_ids)}
+    unsupported = sorted(value for value in found if value not in allowed_numeric)
+    leaked_pii = bool(PII.search(output))
+    injection = bool(INJECTION.search(output))
+    penalties = (0 if supported_ids else 0.35) + min(0.35, len(unsupported) * 0.1)
+    penalties += 0.2 if leaked_pii else 0
+    penalties += 0.1 if injection else 0
+    return NarrativeEvaluation(
+        supported_evidence_ids=supported_ids,
+        unsupported_numbers=unsupported,
+        leaked_pii=leaked_pii,
+        prompt_injection_echo=injection,
+        score=round(max(0.0, 1.0 - penalties), 3),
+    )
+
+
+def _normalize_number(value: str) -> str:
+    return value.replace(",", "").rstrip("%").lstrip("+")

datapilot/insights.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from datapilot.config import Settings
+from datapilot.schemas import Evidence
+
+
+def deterministic_insights(state: dict[str, Any]) -> tuple[list[str], list[str]]:
+    profile = state["profile"]
+    best = state["model_bundle"].results[0]
+    quality = state["quality_issues"]
+    explainability = state["explainability"]
+    important = list(explainability.feature_importance)[:3]
+    summary = [
+        (
+            f"The analysis used {profile.rows:,} rows and {profile.columns:,} columns for a "
+            f"{profile.task_type.value} task targeting '{profile.target}'."
+        ),
+        (
+            f"{best.name} ranked first with training CV {best.primary_metric} {best.primary_score:.3f}; "
+            f"its one-time test score was {best.final_test_score:.3f}."
+        ),
+        (
+            f"{len(quality)} data-quality observations were recorded; "
+            f"{sum(issue.severity.value == 'critical' for issue in quality)} are critical."
+        ),
+    ]
+    if important:
+        summary.append(
+            f"The strongest predictive signals were {', '.join(important)} "
+            f"according to {explainability.method.lower()}."
+        )
+    recommendations = [
+        "Validate performance on fresh, out-of-time data before production deployment.",
+        "Review suspected leakage and identifier columns with a domain owner.",
+        "Monitor input drift and the primary metric after deployment.",
+    ]
+    if profile.missing_rate > 0.1:
+        recommendations.insert(0, "Investigate upstream causes of missing data before retraining.")
+    return summary, recommendations
+
+
+def optional_llm_narrative(
+    state: dict[str, Any], evidence: list[Evidence], settings: Settings
+) -> list[str] | None:
+    """Generate narrative only from bounded evidence; calculations remain deterministic."""
+    if not settings.gemini_api_key:
+        return None
+    try:
+        from google import genai
+
+        client = genai.Client(api_key=settings.gemini_api_key)
+        payload = {
+            "profile": state["profile"].model_dump(),
+            "best_model": state["model_bundle"].results[0].model_dump(),
+            "critic": state["critic"].model_dump(),
+            "evidence": [item.model_dump() for item in evidence[:25]],
+        }
+        prompt = (
+            "You are a senior data scientist. Return exactly four concise markdown bullet points. "
+            "Use only the JSON evidence below. Cite supporting evidence IDs in square brackets. "
+            "Do not add numbers, causal claims, or facts absent from the payload.\n"
+            + json.dumps(payload, default=str)
+        )
+        response = client.models.generate_content(model=settings.gemini_model, contents=prompt)
+        lines = [line.strip("- ").strip() for line in response.text.splitlines() if line.strip()]
+        return lines[:4] or None
+    except Exception:
+        return None
+
+
+def answer_follow_up(run: dict[str, Any], question: str) -> str:
+    lowered = question.lower()
+    if any(token in lowered for token in {"best model", "which model", "winner"}):
+        top = run["model_results"][0]
+        return (
+            f"The best model was **{top['name']}**, with {top['primary_metric']} "
+            f"training-CV **{top['selection_score']:.3f}** and one-time test "
+            f"**{top['final_test_score']:.3f}**."
+        )
+    if any(token in lowered for token in {"feature", "important", "driver"}):
+        importance = run["explainability"]["feature_importance"]
+        top = list(importance.items())[:5]
+        return (
+            "Top predictive features: "
+            + ", ".join(f"**{name}** ({value:.4f})" for name, value in top)
+            + ". These are associations, not causal effects."
+        )
+    if any(token in lowered for token in {"quality", "missing", "leak", "risk"}):
+        issues = run["quality_issues"]
+        if not issues:
+            return "No material quality flags were detected by the configured checks."
+        return "Quality observations: " + "; ".join(item["message"] for item in issues[:6])
+    if any(token in lowered for token in {"metric", "performance", "score"}):
+        top = run["model_results"][0]
+        formatted = ", ".join(
+            f"{key}={value:.3f}" for key, value in top["final_test_metrics"].items()
+        )
+        return f"Selected-model one-time test metrics: {formatted}."
+    return (
+        "I can answer evidence-backed questions about the best model, performance metrics, "
+        "data quality, leakage risk, and feature importance for this run."
+    )

datapilot/jobs.py

@@ -0,0 +1,78 @@
+"""Bounded asynchronous analysis jobs with a production-queue compatible contract."""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+from uuid import uuid4
+
+
+@dataclass
+class Job:
+    job_id: str
+    status: str = "queued"
+    progress: int = 0
+    created_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
+    updated_at: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
+    result: Any | None = None
+    error: str | None = None
+    future: Future[Any] | None = field(default=None, repr=False)
+
+    def public(self) -> dict[str, Any]:
+        return {
+            "job_id": self.job_id,
+            "status": self.status,
+            "progress": self.progress,
+            "created_at": self.created_at,
+            "updated_at": self.updated_at,
+            "result": self.result if self.status == "completed" else None,
+            "error": self.error,
+        }
+
+
+class JobManager:
+    """In-process development queue; replaceable by ARQ/Celery behind the same API."""
+
+    def __init__(self, workers: int = 2) -> None:
+        self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="datapilot")
+        self._jobs: dict[str, Job] = {}
+        self._lock = threading.Lock()
+
+    def submit(self, operation: Callable[[], Any]) -> Job:
+        job = Job(job_id=f"job_{uuid4().hex[:12]}")
+        with self._lock:
+            self._jobs[job.job_id] = job
+        job.future = self._executor.submit(self._run, job, operation)
+        return job
+
+    def get(self, job_id: str) -> Job | None:
+        with self._lock:
+            return self._jobs.get(job_id)
+
+    def cancel(self, job_id: str) -> bool:
+        job = self.get(job_id)
+        if job is None or job.future is None or job.status not in {"queued", "running"}:
+            return False
+        cancelled = job.future.cancel()
+        if cancelled:
+            job.status, job.progress = "cancelled", 0
+            job.updated_at = datetime.now(UTC).isoformat()
+        return cancelled
+
+    @staticmethod
+    def _run(job: Job, operation: Callable[[], Any]) -> None:
+        job.status, job.progress = "running", 10
+        job.updated_at = datetime.now(UTC).isoformat()
+        try:
+            value = operation()
+            job.result = value.model_dump(mode="json") if hasattr(value, "model_dump") else value
+            job.status, job.progress = "completed", 100
+        except Exception:
+            job.status, job.progress = "failed", 100
+            job.error = "Analysis failed. Use the correlation ID in server logs for support."
+        finally:
+            job.updated_at = datetime.now(UTC).isoformat()

datapilot/modeling.py

@@ -0,0 +1,389 @@
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import pandas as pd
+from sklearn.base import BaseEstimator
+from sklearn.compose import ColumnTransformer
+from sklearn.ensemble import (
+    ExtraTreesClassifier,
+    ExtraTreesRegressor,
+    RandomForestClassifier,
+    RandomForestRegressor,
+)
+from sklearn.impute import SimpleImputer
+from sklearn.inspection import permutation_importance
+from sklearn.linear_model import LinearRegression, LogisticRegression
+from sklearn.metrics import (
+    accuracy_score,
+    balanced_accuracy_score,
+    f1_score,
+    mean_absolute_error,
+    mean_squared_error,
+    r2_score,
+    roc_auc_score,
+)
+from sklearn.model_selection import cross_val_score, train_test_split
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+from datapilot.config import Settings
+from datapilot.schemas import (
+    CriticDecision,
+    ExplainabilityResult,
+    ModelFailure,
+    ModelResult,
+    TaskType,
+)
+from datapilot.tuning import tune_random_forest
+
+
+@dataclass
+class TrainingBundle:
+    pipeline: Pipeline
+    results: list[ModelResult]
+    best_model: str
+    test_features: pd.DataFrame
+    test_target: pd.Series
+    retry_number: int
+    failures: list[ModelFailure]
+
+
+logger = logging.getLogger(__name__)
+
+
+def _preprocessor(features: pd.DataFrame, settings: Settings) -> ColumnTransformer:
+    numeric = features.select_dtypes(include=np.number).columns.tolist()
+    categorical = [column for column in features.columns if column not in numeric]
+    numeric_pipeline = Pipeline(
+        [
+            ("imputer", SimpleImputer(strategy="median")),
+            ("scaler", StandardScaler()),
+        ]
+    )
+    categorical_pipeline = Pipeline(
+        [
+            ("imputer", SimpleImputer(strategy="most_frequent")),
+            (
+                "encoder",
+                OneHotEncoder(
+                    handle_unknown="infrequent_if_exist",
+                    min_frequency=2,
+                    max_categories=settings.max_categories_per_feature,
+                    sparse_output=True,
+                ),
+            ),
+        ]
+    )
+    return ColumnTransformer(
+        [
+            ("numeric", numeric_pipeline, numeric),
+            ("categorical", categorical_pipeline, categorical),
+        ],
+        remainder="drop",
+        verbose_feature_names_out=False,
+    )
+
+
+def _candidate_models(task: TaskType, settings: Settings, retry: int) -> dict[str, BaseEstimator]:
+    if task == TaskType.classification:
+        models: dict[str, BaseEstimator] = {
+            "Logistic Regression": LogisticRegression(
+                max_iter=1_000, class_weight="balanced", random_state=settings.random_state
+            ),
+            "Random Forest": RandomForestClassifier(
+                n_estimators=180 + retry * 80,
+                min_samples_leaf=max(1, 2 - retry),
+                class_weight="balanced",
+                n_jobs=-1,
+                random_state=settings.random_state,
+            ),
+            "Extra Trees": ExtraTreesClassifier(
+                n_estimators=180 + retry * 80,
+                class_weight="balanced",
+                n_jobs=-1,
+                random_state=settings.random_state,
+            ),
+        }
+    else:
+        models = {
+            "Linear Regression": LinearRegression(),
+            "Random Forest": RandomForestRegressor(
+                n_estimators=180 + retry * 80,
+                min_samples_leaf=max(1, 2 - retry),
+                n_jobs=-1,
+                random_state=settings.random_state,
+            ),
+            "Extra Trees": ExtraTreesRegressor(
+                n_estimators=180 + retry * 80,
+                n_jobs=-1,
+                random_state=settings.random_state,
+            ),
+        }
+    try:
+        if task == TaskType.classification:
+            from xgboost import XGBClassifier
+
+            models["XGBoost"] = XGBClassifier(
+                n_estimators=160 + retry * 60,
+                max_depth=4 + retry,
+                learning_rate=0.07,
+                eval_metric="logloss",
+                n_jobs=-1,
+                random_state=settings.random_state,
+            )
+        else:
+            from xgboost import XGBRegressor
+
+            models["XGBoost"] = XGBRegressor(
+                n_estimators=160 + retry * 60,
+                max_depth=4 + retry,
+                learning_rate=0.07,
+                n_jobs=-1,
+                random_state=settings.random_state,
+            )
+    except ImportError:
+        pass
+    return models
+
+
+def train_models(
+    frame: pd.DataFrame,
+    target: str,
+    task: TaskType,
+    settings: Settings,
+    retry_number: int = 0,
+) -> TrainingBundle:
+    if target not in frame.columns:
+        raise ValueError(f"Target column '{target}' does not exist.")
+    clean = frame.dropna(subset=[target]).drop_duplicates().reset_index(drop=True)
+    features = clean.drop(columns=[target]).copy()
+    labels = clean[target].copy()
+    if features.empty:
+        raise ValueError("No usable feature columns remain after removing the target.")
+    if labels.nunique(dropna=True) < 2:
+        raise ValueError("The target must contain at least two distinct non-null values.")
+    categorical = features.select_dtypes(exclude=np.number)
+    estimated_width = len(features.select_dtypes(include=np.number).columns) + sum(
+        min(int(categorical[column].nunique(dropna=True)), settings.max_categories_per_feature)
+        for column in categorical.columns
+    )
+    if estimated_width > settings.max_encoded_features:
+        raise ValueError(
+            f"Estimated encoded width {estimated_width:,} exceeds the safe limit of "
+            f"{settings.max_encoded_features:,}. Reduce high-cardinality columns or increase "
+            "MAX_ENCODED_FEATURES after reviewing memory capacity."
+        )
+    stratify = (
+        labels if task == TaskType.classification and labels.value_counts().min() >= 2 else None
+    )
+    x_train, x_test, y_train, y_test = train_test_split(
+        features,
+        labels,
+        test_size=settings.test_size,
+        random_state=settings.random_state,
+        stratify=stratify,
+    )
+    results: list[ModelResult] = []
+    failures: list[ModelFailure] = []
+    cv = min(5, max(2, int(len(x_train) / 20)))
+    if task == TaskType.classification:
+        smallest_class = int(y_train.value_counts().min())
+        cv = min(cv, smallest_class) if smallest_class >= 2 else 0
+        scoring = "balanced_accuracy"
+    else:
+        scoring = "r2"
+    if cv < 2:
+        raise ValueError("The training partition is too small for reliable cross-validation.")
+
+    candidates = _candidate_models(task, settings, retry_number)
+    tuned_parameters = tune_random_forest(
+        task,
+        _preprocessor(x_train, settings),
+        x_train,
+        y_train,
+        settings,
+        cv,
+    )
+    if tuned_parameters:
+        candidates["Random Forest"].set_params(**tuned_parameters)
+
+    for name, estimator in candidates.items():
+        pipeline = Pipeline(
+            [("preprocessor", _preprocessor(x_train, settings)), ("model", estimator)]
+        )
+        started = time.perf_counter()
+        try:
+            cv_scores = (
+                cross_val_score(pipeline, x_train, y_train, scoring=scoring, cv=cv, n_jobs=1)
+                if cv >= 2
+                else np.array([])
+            )
+            result = ModelResult(
+                name=name,
+                primary_metric="balanced_accuracy" if task == TaskType.classification else "r2",
+                primary_score=float(cv_scores.mean()),
+                metrics={},
+                cross_validation_mean=float(cv_scores.mean()) if len(cv_scores) else None,
+                cross_validation_std=float(cv_scores.std()) if len(cv_scores) else None,
+                training_seconds=round(time.perf_counter() - started, 3),
+                selection_score=float(cv_scores.mean()) if len(cv_scores) else None,
+            )
+            results.append(result)
+        except Exception as exc:
+            logger.warning(
+                "Candidate %s failed during cross-validation: %s", name, type(exc).__name__
+            )
+            failures.append(
+                ModelFailure(
+                    name=name,
+                    stage="cross_validation",
+                    exception_category=type(exc).__name__,
+                    sanitized_error="Candidate failed during cross-validation; inspect structured logs.",
+                    training_seconds=round(time.perf_counter() - started, 3),
+                    expected=isinstance(exc, (ValueError, TypeError)),
+                )
+            )
+    if not results:
+        raise RuntimeError("Every candidate model failed; review the data types and target.")
+    results.sort(key=lambda item: item.selection_score or float("-inf"), reverse=True)
+    best = results[0]
+    final_pipeline = Pipeline(
+        [
+            ("preprocessor", _preprocessor(x_train, settings)),
+            ("model", candidates[best.name]),
+        ]
+    )
+    final_pipeline.fit(x_train, y_train)
+    predictions = final_pipeline.predict(x_test)
+    final_metrics = _metrics(task, y_test, predictions, final_pipeline, x_test)
+    final_score = (
+        final_metrics["balanced_accuracy"]
+        if task == TaskType.classification
+        else final_metrics["r2"]
+    )
+    best.final_test_score = float(final_score)
+    best.final_test_metrics = final_metrics
+    best.metrics = final_metrics
+    return TrainingBundle(
+        pipeline=final_pipeline,
+        results=results,
+        best_model=best.name,
+        test_features=x_test,
+        test_target=y_test,
+        retry_number=retry_number,
+        failures=failures,
+    )
+
+
+def _metrics(
+    task: TaskType,
+    truth: pd.Series,
+    predictions: np.ndarray,
+    pipeline: Pipeline,
+    features: pd.DataFrame,
+) -> dict[str, float]:
+    if task == TaskType.classification:
+        metrics = {
+            "accuracy": round(float(accuracy_score(truth, predictions)), 4),
+            "balanced_accuracy": round(float(balanced_accuracy_score(truth, predictions)), 4),
+            "f1_weighted": round(float(f1_score(truth, predictions, average="weighted")), 4),
+        }
+        if truth.nunique() == 2 and hasattr(pipeline, "predict_proba"):
+            probabilities = pipeline.predict_proba(features)[:, 1]
+            metrics["roc_auc"] = round(float(roc_auc_score(truth, probabilities)), 4)
+        return metrics
+    return {
+        "r2": round(float(r2_score(truth, predictions)), 4),
+        "rmse": round(float(mean_squared_error(truth, predictions) ** 0.5), 4),
+        "mae": round(float(mean_absolute_error(truth, predictions)), 4),
+    }
+
+
+def critic_decision(bundle: TrainingBundle, task: TaskType, settings: Settings) -> CriticDecision:
+    best = bundle.results[0]
+    threshold = (
+        settings.min_classification_score
+        if task == TaskType.classification
+        else settings.min_regression_score
+    )
+    reasons: list[str] = []
+    quality_score = best.cross_validation_mean if best.cross_validation_mean is not None else -1.0
+    if quality_score < threshold:
+        reasons.append(
+            f"Training CV {best.primary_metric} {quality_score:.3f} is below {threshold:.3f}."
+        )
+    if (
+        best.cross_validation_mean is not None
+        and best.final_test_score is not None
+        and abs(best.final_test_score - best.cross_validation_mean) > 0.2
+    ):
+        reasons.append("Holdout and cross-validation scores diverge by more than 0.20.")
+    approved = not reasons or bundle.retry_number >= settings.max_critic_retries
+    if not reasons:
+        reasons.append("Performance and validation consistency passed the configured quality gate.")
+    elif approved:
+        reasons.append("Retry budget exhausted; result is retained with an explicit limitation.")
+    return CriticDecision(
+        approved=approved,
+        score=quality_score,
+        threshold=threshold,
+        reasons=reasons,
+        retry_number=bundle.retry_number,
+    )
+
+
+def explain_model(bundle: TrainingBundle) -> ExplainabilityResult:
+    sample_size = min(300, len(bundle.test_features))
+    features = bundle.test_features.iloc[:sample_size]
+    target = bundle.test_target.iloc[:sample_size]
+    try:
+        transformed = bundle.pipeline.named_steps["preprocessor"].transform(features)
+        transformed_names = bundle.pipeline.named_steps["preprocessor"].get_feature_names_out()
+        estimator = bundle.pipeline.named_steps["model"]
+        import shap
+
+        explainer = shap.Explainer(estimator, transformed)
+        values = explainer(transformed)
+        raw = np.asarray(values.values)
+        if raw.ndim == 3:
+            raw = np.abs(raw).mean(axis=(0, 2))
+        else:
+            raw = np.abs(raw).mean(axis=0)
+        importance = _top_importance(transformed_names, raw)
+        return ExplainabilityResult(
+            method="SHAP",
+            feature_importance=importance,
+            caveats=["SHAP values explain this fitted model, not causal effects."],
+        )
+    except Exception:
+        permutation = permutation_importance(
+            bundle.pipeline,
+            features,
+            target,
+            n_repeats=5,
+            random_state=42,
+            n_jobs=1,
+        )
+        importance = _top_importance(features.columns, np.abs(permutation.importances_mean))
+        return ExplainabilityResult(
+            method="Permutation importance",
+            feature_importance=importance,
+            caveats=[
+                "Permutation importance can dilute importance among correlated features.",
+                "Feature importance is predictive, not causal.",
+            ],
+        )
+
+
+def _top_importance(names: Any, values: np.ndarray, limit: int = 15) -> dict[str, float]:
+    pairs = sorted(
+        zip([str(name) for name in names], values.tolist(), strict=False),
+        key=lambda item: item[1],
+        reverse=True,
+    )[:limit]
+    return {name: round(float(value), 6) for name, value in pairs}

datapilot/observability.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+from datapilot.config import Settings
+
+
+def log_to_mlflow(payload: dict[str, Any], settings: Settings) -> bool:
+    """Record a run when MLflow is enabled; local demos remain dependency-light."""
+    if not settings.enable_mlflow:
+        return False
+    try:
+        import mlflow
+
+        mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
+        mlflow.set_experiment("datapilot-ai")
+        best = payload["model_results"][0]
+        with mlflow.start_run(run_name=payload["run_id"]):
+            mlflow.log_params(
+                {
+                    "dataset": payload["dataset_name"],
+                    "task_type": payload["profile"]["task_type"],
+                    "target": payload["profile"]["target"],
+                    "selected_model": payload["best_model"],
+                }
+            )
+            mlflow.log_metrics({key: float(value) for key, value in best["metrics"].items()})
+        return True
+    except Exception:
+        return False

datapilot/persistence.py

@@ -0,0 +1,91 @@
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import DateTime, String, Text, create_engine
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker
+
+from datapilot.config import Settings
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class AnalysisRun(Base):
+    __tablename__ = "analysis_runs"
+
+    run_id: Mapped[str] = mapped_column(String(64), primary_key=True)
+    dataset_name: Mapped[str] = mapped_column(String(255))
+    status: Mapped[str] = mapped_column(String(32), index=True)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    payload_json: Mapped[str] = mapped_column(Text)
+
+
+class RunStore:
+    """SQLite locally; set DATABASE_URL to PostgreSQL in production."""
+
+    def __init__(self, settings: Settings):
+        self.engine = create_engine(settings.database_url, future=True)
+        Base.metadata.create_all(self.engine)
+        self.sessions = sessionmaker(self.engine, expire_on_commit=False)
+
+    def save(self, run_id: str, dataset_name: str, status: str, payload: dict[str, Any]) -> None:
+        now = datetime.now(UTC)
+        with self.sessions.begin() as session:
+            record = session.get(AnalysisRun, run_id)
+            if record is None:
+                record = AnalysisRun(
+                    run_id=run_id,
+                    dataset_name=dataset_name,
+                    status=status,
+                    created_at=now,
+                    updated_at=now,
+                    payload_json=json.dumps(payload, default=str),
+                )
+                session.add(record)
+            else:
+                record.status = status
+                record.updated_at = now
+                record.payload_json = json.dumps(payload, default=str)
+
+    def get(self, run_id: str) -> dict[str, Any] | None:
+        with self.sessions() as session:
+            record = session.get(AnalysisRun, run_id)
+            return json.loads(record.payload_json) if record else None
+
+    def list_recent(self, limit: int = 20) -> list[dict[str, Any]]:
+        from sqlalchemy import select
+
+        with self.sessions() as session:
+            records = session.scalars(
+                select(AnalysisRun).order_by(AnalysisRun.created_at.desc()).limit(limit)
+            )
+            return [
+                {
+                    "run_id": record.run_id,
+                    "dataset_name": record.dataset_name,
+                    "status": record.status,
+                    "created_at": record.created_at.isoformat(),
+                }
+                for record in records
+            ]
+
+
+class ArtifactStore:
+    """Local artifact storage with an interface that can be replaced by S3/MinIO."""
+
+    def __init__(self, root: Path):
+        self.root = root
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def run_directory(self, run_id: str) -> Path:
+        directory = (self.root / run_id).resolve()
+        if self.root.resolve() not in directory.parents:
+            raise ValueError("Invalid run identifier.")
+        directory.mkdir(parents=True, exist_ok=True)
+        return directory

datapilot/quality.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+
+import re
+from uuid import uuid4
+
+import numpy as np
+import pandas as pd
+
+from datapilot.schemas import (
+    DatasetProfile,
+    Evidence,
+    QualityIssue,
+    Severity,
+    TaskType,
+)
+
+LEAKAGE_PATTERNS = re.compile(
+    r"(target|label|outcome|result|prediction|predicted|probability|score)$",
+    re.IGNORECASE,
+)
+
+
+def infer_task_type(target: pd.Series) -> TaskType:
+    unique = int(target.nunique(dropna=True))
+    if (
+        not pd.api.types.is_numeric_dtype(target)
+        or pd.api.types.is_bool_dtype(target)
+        or unique <= 20
+        or unique / max(len(target), 1) < 0.05
+    ):
+        return TaskType.classification
+    return TaskType.regression
+
+
+def build_profile(frame: pd.DataFrame, target: str) -> DatasetProfile:
+    if target not in frame.columns:
+        raise ValueError(f"Target column '{target}' is not present.")
+    numeric = frame.select_dtypes(include=np.number).columns.tolist()
+    categorical = frame.select_dtypes(include=["object", "category", "bool"]).columns.tolist()
+    datetime = frame.select_dtypes(include=["datetime", "datetimetz"]).columns.tolist()
+    missing_cells = int(frame.isna().sum().sum())
+    return DatasetProfile(
+        rows=len(frame),
+        columns=len(frame.columns),
+        numeric_columns=numeric,
+        categorical_columns=categorical,
+        datetime_columns=datetime,
+        duplicate_rows=int(frame.duplicated().sum()),
+        missing_cells=missing_cells,
+        missing_rate=round(missing_cells / max(frame.size, 1), 4),
+        memory_mb=round(frame.memory_usage(deep=True).sum() / 1_048_576, 3),
+        target=target,
+        task_type=infer_task_type(frame[target]),
+        target_cardinality=int(frame[target].nunique(dropna=True)),
+    )
+
+
+def audit_quality(
+    frame: pd.DataFrame, profile: DatasetProfile
+) -> tuple[list[QualityIssue], list[Evidence]]:
+    issues: list[QualityIssue] = []
+    evidence: list[Evidence] = []
+
+    def add_evidence(claim: str, metric: str, value: object, source: str, method: str) -> str:
+        evidence_id = f"EV-{uuid4().hex[:8].upper()}"
+        evidence.append(
+            Evidence(
+                evidence_id=evidence_id,
+                claim=claim,
+                metric=metric,
+                value=value,
+                source=source,
+                method=method,
+            )
+        )
+        return evidence_id
+
+    missing_id = add_evidence(
+        "Dataset missingness was measured across all cells.",
+        "missing_rate",
+        profile.missing_rate,
+        "uploaded_dataset",
+        "pandas.isna",
+    )
+    if profile.missing_rate > 0.2:
+        issues.append(
+            QualityIssue(
+                code="HIGH_MISSINGNESS",
+                severity=Severity.critical,
+                message=f"{profile.missing_rate:.1%} of dataset cells are missing.",
+                evidence_ids=[missing_id],
+            )
+        )
+    elif profile.missing_rate > 0:
+        issues.append(
+            QualityIssue(
+                code="MISSING_VALUES",
+                severity=Severity.warning,
+                message=f"{profile.missing_rate:.1%} of dataset cells are missing.",
+                evidence_ids=[missing_id],
+            )
+        )
+
+    duplicate_id = add_evidence(
+        "Exact duplicate rows were counted before splitting.",
+        "duplicate_rows",
+        profile.duplicate_rows,
+        "uploaded_dataset",
+        "pandas.duplicated",
+    )
+    if profile.duplicate_rows:
+        issues.append(
+            QualityIssue(
+                code="DUPLICATE_ROWS",
+                severity=Severity.warning,
+                message=f"{profile.duplicate_rows:,} exact duplicate rows can bias validation.",
+                evidence_ids=[duplicate_id],
+            )
+        )
+
+    target = frame[profile.target]
+    target_missing = int(target.isna().sum())
+    target_missing_id = add_evidence(
+        "Rows with missing labels cannot be used for supervised training.",
+        "missing_target_rows",
+        target_missing,
+        f"column:{profile.target}",
+        "pandas.isna",
+    )
+    if target_missing:
+        issues.append(
+            QualityIssue(
+                code="MISSING_TARGET",
+                severity=Severity.critical,
+                column=profile.target,
+                message=f"{target_missing:,} rows have no target value and will be excluded.",
+                evidence_ids=[target_missing_id],
+            )
+        )
+
+    if profile.task_type == TaskType.classification:
+        distribution = target.value_counts(normalize=True, dropna=True)
+        minority_share = float(distribution.min()) if not distribution.empty else 0.0
+        imbalance_id = add_evidence(
+            "Class imbalance was measured using the minority-class share.",
+            "minority_class_share",
+            round(minority_share, 4),
+            f"column:{profile.target}",
+            "normalized value counts",
+        )
+        if minority_share < 0.1:
+            issues.append(
+                QualityIssue(
+                    code="CLASS_IMBALANCE",
+                    severity=Severity.warning,
+                    column=profile.target,
+                    message=f"Minority class represents only {minority_share:.1%} of labeled rows.",
+                    evidence_ids=[imbalance_id],
+                )
+            )
+
+    feature_frame = frame.drop(columns=[profile.target])
+    for column in feature_frame.columns:
+        normalized = column.strip().lower()
+        leakage_risk = bool(LEAKAGE_PATTERNS.search(normalized))
+        if feature_frame[column].nunique(dropna=True) == len(feature_frame):
+            leakage_risk = leakage_risk or normalized.endswith(("_id", "id"))
+        if leakage_risk:
+            evidence_id = add_evidence(
+                "A feature name or cardinality pattern may reveal the target or row identity.",
+                "suspected_leakage_feature",
+                column,
+                f"column:{column}",
+                "name and cardinality heuristic",
+            )
+            issues.append(
+                QualityIssue(
+                    code="LEAKAGE_RISK",
+                    severity=Severity.warning,
+                    column=column,
+                    message=f"'{column}' may leak target or row identity; review before deployment.",
+                    evidence_ids=[evidence_id],
+                )
+            )
+
+    numeric = feature_frame.select_dtypes(include=np.number)
+    for column in numeric.columns:
+        series = numeric[column].dropna()
+        if len(series) < 8:
+            continue
+        q1, q3 = series.quantile([0.25, 0.75])
+        iqr = q3 - q1
+        if iqr == 0:
+            continue
+        outlier_rate = float(((series < q1 - 1.5 * iqr) | (series > q3 + 1.5 * iqr)).mean())
+        if outlier_rate > 0.05:
+            evidence_id = add_evidence(
+                "Potential outliers were detected with the 1.5×IQR rule.",
+                "outlier_rate",
+                round(outlier_rate, 4),
+                f"column:{column}",
+                "Tukey IQR",
+            )
+            issues.append(
+                QualityIssue(
+                    code="OUTLIER_RATE",
+                    severity=Severity.info,
+                    column=column,
+                    message=f"'{column}' has {outlier_rate:.1%} potential outliers.",
+                    evidence_ids=[evidence_id],
+                )
+            )
+
+    return issues, evidence
+
+
+def drift_report(reference: pd.DataFrame, current: pd.DataFrame) -> list[dict[str, object]]:
+    """Population stability index for numeric columns shared by two datasets."""
+    reports: list[dict[str, object]] = []
+    shared = reference.select_dtypes(include=np.number).columns.intersection(
+        current.select_dtypes(include=np.number).columns
+    )
+    for column in shared:
+        baseline = reference[column].dropna()
+        observed = current[column].dropna()
+        if baseline.nunique() < 2 or observed.empty:
+            continue
+        edges = np.unique(baseline.quantile(np.linspace(0, 1, 11)).to_numpy())
+        if len(edges) < 3:
+            continue
+        expected_counts, _ = np.histogram(baseline, bins=edges)
+        actual_counts, _ = np.histogram(observed, bins=edges)
+        expected = np.clip(expected_counts / max(expected_counts.sum(), 1), 1e-6, None)
+        actual = np.clip(actual_counts / max(actual_counts.sum(), 1), 1e-6, None)
+        psi = float(np.sum((actual - expected) * np.log(actual / expected)))
+        reports.append(
+            {
+                "column": column,
+                "psi": round(psi, 4),
+                "status": "high" if psi >= 0.25 else "moderate" if psi >= 0.1 else "stable",
+            }
+        )
+    return reports

datapilot/reports.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import html
+import json
+from pathlib import Path
+from typing import Any
+
+import joblib
+
+
+def export_artifacts(
+    run_id: str,
+    state: dict[str, Any],
+    run_directory: Path,
+) -> dict[str, str]:
+    run_directory.mkdir(parents=True, exist_ok=True)
+    bundle = state["model_bundle"]
+    summary = state["summary_payload"]
+
+    pipeline_path = run_directory / "model_pipeline.joblib"
+    joblib.dump(bundle.pipeline, pipeline_path)
+
+    metrics_path = run_directory / "metrics.json"
+    metrics_path.write_text(json.dumps(summary, indent=2, default=str), encoding="utf-8")
+
+    model_card_path = run_directory / "MODEL_CARD.md"
+    model_card_path.write_text(_model_card(summary), encoding="utf-8")
+
+    report_path = run_directory / "analysis_report.html"
+    report_path.write_text(_html_report(summary), encoding="utf-8")
+
+    requirements_path = run_directory / "reproduction.json"
+    requirements_path.write_text(
+        json.dumps(
+            {
+                "run_id": run_id,
+                "random_state": state["settings"].random_state,
+                "test_size": state["settings"].test_size,
+                "target": summary["profile"]["target"],
+                "task_type": summary["profile"]["task_type"],
+                "best_model": summary["best_model"],
+            },
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+    return {
+        "pipeline": str(pipeline_path),
+        "metrics": str(metrics_path),
+        "model_card": str(model_card_path),
+        "report": str(report_path),
+        "reproduction": str(requirements_path),
+    }
+
+
+def _model_card(run: dict[str, Any]) -> str:
+    best = run["model_results"][0]
+    profile = run["profile"]
+    issues = (
+        "\n".join(f"- {item['message']}" for item in run["quality_issues"]) or "- None detected"
+    )
+    return f"""# Model Card — {run["dataset_name"]}
+
+## Model details
+
+- Run ID: `{run["run_id"]}`
+- Task: {profile["task_type"]}
+- Target: `{profile["target"]}`
+- Selected model: **{run["best_model"]}**
+- Training-CV selection metric: `{best["primary_metric"]} = {best["selection_score"]:.4f}`
+- One-time untouched test metric: `{best["primary_metric"]} = {best["final_test_score"]:.4f}`
+- Training rows before split: {profile["rows"]:,}
+
+## Intended use
+
+Exploratory decision support and portfolio demonstration. Validate with domain-specific,
+out-of-time data before any consequential or production use.
+
+## Evaluation
+
+```json
+{json.dumps(best["final_test_metrics"], indent=2)}
+```
+
+## Data-quality observations
+
+{issues}
+
+## Explainability
+
+Method: **{run["explainability"]["method"]}**. Importance values are predictive associations,
+not evidence of causation.
+
+## Limitations
+
+- Results depend on the uploaded dataset and chosen target.
+- Automated task inference can be wrong; a domain owner should confirm the objective.
+- Fairness, privacy, and legal review are outside the automatic approval gate.
+"""
+
+
+def _html_report(run: dict[str, Any]) -> str:
+    best = run["model_results"][0]
+    summary_items = "".join(f"<li>{html.escape(item)}</li>" for item in run["executive_summary"])
+    recommendations = "".join(f"<li>{html.escape(item)}</li>" for item in run["recommendations"])
+    issues = (
+        "".join(
+            f"<tr><td>{html.escape(item['severity'])}</td><td>{html.escape(item['code'])}</td>"
+            f"<td>{html.escape(item['message'])}</td></tr>"
+            for item in run["quality_issues"]
+        )
+        or "<tr><td colspan='3'>No material flags</td></tr>"
+    )
+    metrics = "".join(
+        f"<tr><td>{html.escape(name)}</td><td>{value:.4f}</td></tr>"
+        for name, value in best["final_test_metrics"].items()
+    )
+    return f"""<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
+<title>DataPilot AI report</title>
+<style>
+body{{font-family:Inter,system-ui,sans-serif;max-width:1000px;margin:40px auto;padding:0 24px;color:#172033}}
+h1{{color:#5537d8}} .hero{{background:#f4f1ff;border:1px solid #d9d0ff;padding:24px;border-radius:18px}}
+.grid{{display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:16px;margin:20px 0}}
+.card{{border:1px solid #e2e6ef;border-radius:14px;padding:18px}} table{{border-collapse:collapse;width:100%}}
+td,th{{border-bottom:1px solid #e2e6ef;padding:10px;text-align:left}} small{{color:#667085}}
+</style></head><body>
+<div class="hero"><h1>DataPilot AI Analysis Report</h1>
+<p>{html.escape(run["dataset_name"])} · Run {html.escape(run["run_id"])}</p></div>
+<div class="grid">
+<div class="card"><small>Selected model</small><h2>{html.escape(run["best_model"])}</h2></div>
+<div class="card"><small>Test {html.escape(best["primary_metric"])}</small><h2>{best["final_test_score"]:.3f}</h2></div>
+<div class="card"><small>Rows analyzed</small><h2>{run["profile"]["rows"]:,}</h2></div>
+</div>
+<h2>Executive findings</h2><ul>{summary_items}</ul>
+<h2>Evaluation</h2><table><tr><th>Metric</th><th>Value</th></tr>{metrics}</table>
+<h2>Data quality</h2><table><tr><th>Severity</th><th>Code</th><th>Observation</th></tr>{issues}</table>
+<h2>Recommendations</h2><ol>{recommendations}</ol>
+<p><small>Generated from computed evidence. Predictive findings do not establish causality.</small></p>
+</body></html>"""

datapilot/safety.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+import ast
+
+BLOCKED_NODES = (
+    ast.Import,
+    ast.ImportFrom,
+    ast.Global,
+    ast.Nonlocal,
+    ast.With,
+    ast.AsyncWith,
+    ast.Try,
+    ast.Raise,
+    ast.ClassDef,
+    ast.FunctionDef,
+    ast.AsyncFunctionDef,
+)
+BLOCKED_CALLS = {
+    "eval",
+    "exec",
+    "compile",
+    "open",
+    "input",
+    "__import__",
+    "breakpoint",
+    "getattr",
+    "setattr",
+    "delattr",
+}
+
+
+class UnsafeCodeError(ValueError):
+    pass
+
+
+def validate_generated_expression(expression: str, max_characters: int = 2_000) -> ast.Expression:
+    """Validate a calculation expression before it is sent to an isolated worker.
+
+    DataPilot's primary workflow does not execute LLM-generated Python. This validator
+    exists for an optional restricted calculation worker and accepts expressions only.
+    """
+    if len(expression) > max_characters:
+        raise UnsafeCodeError("Expression exceeds the configured size limit.")
+    try:
+        tree = ast.parse(expression, mode="eval")
+    except SyntaxError as exc:
+        raise UnsafeCodeError("Expression is not valid Python.") from exc
+    for node in ast.walk(tree):
+        if isinstance(node, BLOCKED_NODES):
+            raise UnsafeCodeError(f"Blocked syntax: {type(node).__name__}.")
+        if isinstance(node, ast.Attribute) and node.attr.startswith("__"):
+            raise UnsafeCodeError("Dunder attribute access is blocked.")
+        if isinstance(node, ast.Name) and node.id.startswith("__"):
+            raise UnsafeCodeError("Dunder names are blocked.")
+        if isinstance(node, ast.Call):
+            if isinstance(node.func, ast.Name) and node.func.id in BLOCKED_CALLS:
+                raise UnsafeCodeError(f"Blocked call: {node.func.id}.")
+            if isinstance(node.func, ast.Attribute) and node.func.attr in BLOCKED_CALLS:
+                raise UnsafeCodeError(f"Blocked call: {node.func.attr}.")
+    return tree

datapilot/schemas.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class TaskType(StrEnum):
+    classification = "classification"
+    regression = "regression"
+
+
+class Severity(StrEnum):
+    info = "info"
+    warning = "warning"
+    critical = "critical"
+
+
+class Evidence(BaseModel):
+    evidence_id: str
+    claim: str
+    metric: str
+    value: float | int | str
+    source: str
+    method: str
+
+
+class QualityIssue(BaseModel):
+    code: str
+    severity: Severity
+    column: str | None = None
+    message: str
+    evidence_ids: list[str] = Field(default_factory=list)
+
+
+class DatasetProfile(BaseModel):
+    rows: int
+    columns: int
+    numeric_columns: list[str]
+    categorical_columns: list[str]
+    datetime_columns: list[str]
+    duplicate_rows: int
+    missing_cells: int
+    missing_rate: float
+    memory_mb: float
+    target: str
+    task_type: TaskType
+    target_cardinality: int
+
+
+class AnalysisPlan(BaseModel):
+    objective: str
+    target: str
+    task_type: TaskType
+    primary_metric: str
+    validation_strategy: str
+    candidate_models: list[str]
+    risk_controls: list[str]
+
+
+class ModelResult(BaseModel):
+    name: str
+    primary_metric: str
+    primary_score: float
+    metrics: dict[str, float]
+    cross_validation_mean: float | None = None
+    cross_validation_std: float | None = None
+    training_seconds: float
+    selection_score: float | None = None
+    final_test_score: float | None = None
+    final_test_metrics: dict[str, float] = Field(default_factory=dict)
+
+
+class ModelFailure(BaseModel):
+    name: str
+    stage: str
+    exception_category: str
+    sanitized_error: str
+    training_seconds: float
+    expected: bool = False
+
+
+class CriticDecision(BaseModel):
+    approved: bool
+    score: float
+    threshold: float
+    reasons: list[str]
+    retry_number: int
+
+
+class ExplainabilityResult(BaseModel):
+    method: str
+    feature_importance: dict[str, float]
+    caveats: list[str]
+
+
+class RunSummary(BaseModel):
+    run_id: str
+    status: str
+    dataset_name: str
+    profile: DatasetProfile
+    plan: AnalysisPlan
+    quality_issues: list[QualityIssue]
+    evidence: list[Evidence]
+    model_results: list[ModelResult]
+    model_failures: list[ModelFailure] = Field(default_factory=list)
+    best_model: str
+    critic: CriticDecision
+    explainability: ExplainabilityResult
+    executive_summary: list[str]
+    recommendations: list[str]
+    artifacts: dict[str, str]
+    trace: list[dict[str, Any]]

datapilot/tuning.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
+from sklearn.model_selection import cross_val_score
+from sklearn.pipeline import Pipeline
+
+from datapilot.config import Settings
+from datapilot.schemas import TaskType
+
+
+def tune_random_forest(
+    task: TaskType,
+    preprocessor: Any,
+    features: pd.DataFrame,
+    target: pd.Series,
+    settings: Settings,
+    cv: int,
+) -> dict[str, Any]:
+    """Run a bounded Optuna study when the optional AutoML extra is installed."""
+    if settings.optuna_trials <= 0 or cv < 2:
+        return {}
+    try:
+        import optuna
+    except ImportError:
+        return {}
+
+    optuna.logging.set_verbosity(optuna.logging.WARNING)
+    scoring = "balanced_accuracy" if task == TaskType.classification else "r2"
+
+    def objective(trial):
+        parameters = {
+            "n_estimators": trial.suggest_int("n_estimators", 120, 360, step=60),
+            "max_depth": trial.suggest_int("max_depth", 3, 14),
+            "min_samples_leaf": trial.suggest_int("min_samples_leaf", 1, 6),
+            "max_features": trial.suggest_categorical("max_features", ["sqrt", "log2", 0.8]),
+        }
+        common = {
+            **parameters,
+            "n_jobs": -1,
+            "random_state": settings.random_state,
+        }
+        if task == TaskType.classification:
+            estimator = RandomForestClassifier(**common, class_weight="balanced")
+        else:
+            estimator = RandomForestRegressor(**common)
+        pipeline = Pipeline([("preprocessor", preprocessor), ("model", estimator)])
+        scores = cross_val_score(
+            pipeline,
+            features,
+            target,
+            cv=cv,
+            scoring=scoring,
+            n_jobs=1,
+        )
+        return float(scores.mean())
+
+    study = optuna.create_study(direction="maximize")
+    study.optimize(
+        objective,
+        n_trials=settings.optuna_trials,
+        timeout=90,
+        show_progress_bar=False,
+    )
+    return study.best_params

datapilot/workflow.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import time
+from typing import Any, TypedDict
+from uuid import uuid4
+
+import numpy as np
+import pandas as pd
+from langgraph.graph import END, START, StateGraph
+
+from datapilot.config import Settings, get_settings
+from datapilot.data import duckdb_overview
+from datapilot.insights import deterministic_insights, optional_llm_narrative
+from datapilot.modeling import (
+    TrainingBundle,
+    critic_decision,
+    explain_model,
+    train_models,
+)
+from datapilot.observability import log_to_mlflow
+from datapilot.persistence import ArtifactStore, RunStore
+from datapilot.quality import audit_quality, build_profile
+from datapilot.reports import export_artifacts
+from datapilot.schemas import AnalysisPlan, RunSummary, TaskType
+
+
+class AgentState(TypedDict, total=False):
+    run_id: str
+    dataset_name: str
+    frame: pd.DataFrame
+    target: str
+    settings: Settings
+    profile: Any
+    quality_issues: list[Any]
+    evidence: list[Any]
+    eda: dict[str, Any]
+    statistics: dict[str, Any]
+    plan: AnalysisPlan
+    feature_plan: dict[str, Any]
+    model_bundle: TrainingBundle
+    critic: Any
+    explainability: Any
+    executive_summary: list[str]
+    recommendations: list[str]
+    summary_payload: dict[str, Any]
+    artifacts: dict[str, str]
+    trace: list[dict[str, Any]]
+
+
+def _trace(state: AgentState, agent: str, started: float, detail: str) -> list[dict[str, Any]]:
+    trace = list(state.get("trace", []))
+    trace.append(
+        {
+            "agent": agent,
+            "status": "completed",
+            "duration_seconds": round(time.perf_counter() - started, 3),
+            "detail": detail,
+        }
+    )
+    return trace
+
+
+def data_quality_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    profile = build_profile(state["frame"], state["target"])
+    issues, evidence = audit_quality(state["frame"], profile)
+    return {
+        "profile": profile,
+        "quality_issues": issues,
+        "evidence": evidence,
+        "trace": _trace(
+            state, "Data Quality Agent", started, f"Recorded {len(issues)} quality observations."
+        ),
+    }
+
+
+def eda_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    frame = state["frame"]
+    overview = duckdb_overview(frame)
+    overview["numeric_summary"] = (
+        frame.select_dtypes(include=np.number).describe().round(4).to_dict()
+    )
+    overview["categorical_cardinality"] = {
+        column: int(frame[column].nunique(dropna=True))
+        for column in frame.select_dtypes(exclude=np.number).columns
+    }
+    return {
+        "eda": overview,
+        "trace": _trace(state, "EDA Agent", started, "Computed DuckDB-backed dataset overview."),
+    }
+
+
+def statistical_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    frame = state["frame"]
+    target = state["target"]
+    numeric = frame.select_dtypes(include=np.number)
+    correlations: dict[str, float] = {}
+    if target in numeric.columns and len(numeric.columns) > 1:
+        correlations = (
+            numeric.corr(numeric_only=True)[target]
+            .drop(labels=[target])
+            .abs()
+            .sort_values(ascending=False)
+            .head(10)
+            .round(4)
+            .to_dict()
+        )
+    statistics = {
+        "top_absolute_target_correlations": correlations,
+        "target_distribution": frame[target].value_counts(dropna=False).head(20).to_dict(),
+    }
+    return {
+        "statistics": statistics,
+        "trace": _trace(
+            state,
+            "Statistical Analysis Agent",
+            started,
+            "Measured target distribution and associations.",
+        ),
+    }
+
+
+def planning_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    profile = state["profile"]
+    if profile.task_type == TaskType.classification:
+        metric = "balanced_accuracy"
+        candidates = [
+            "Logistic Regression",
+            "Random Forest",
+            "Extra Trees",
+            "Histogram Gradient Boosting",
+            "XGBoost (when installed)",
+        ]
+    else:
+        metric = "r2"
+        candidates = [
+            "Linear Regression",
+            "Random Forest",
+            "Extra Trees",
+            "Histogram Gradient Boosting",
+            "XGBoost (when installed)",
+        ]
+    plan = AnalysisPlan(
+        objective=f"Predict '{profile.target}' and produce reproducible, evidence-backed insights.",
+        target=profile.target,
+        task_type=profile.task_type,
+        primary_metric=metric,
+        validation_strategy="Training-only stratified cross-validation; untouched final test evaluation"
+        if profile.task_type == TaskType.classification
+        else "Training-only cross-validation; untouched final test evaluation",
+        candidate_models=candidates,
+        risk_controls=[
+            "Drop rows with missing target before split",
+            "Fit imputers, encoders, and scalers on training folds only",
+            "Flag leakage-like names and identifier cardinality",
+            "Require critic quality gate before explanation",
+        ],
+    )
+    return {
+        "plan": plan,
+        "trace": _trace(state, "Planning Agent", started, f"Selected {metric} as primary metric."),
+    }
+
+
+def feature_engineering_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    profile = state["profile"]
+    feature_plan = {
+        "numeric": "Median imputation followed by standard scaling",
+        "categorical": "Most-frequent imputation followed by unknown-safe one-hot encoding",
+        "fit_scope": "Preprocessing is fitted inside each sklearn Pipeline after splitting",
+        "dropped": ["exact duplicate rows", "rows with missing target"],
+        "feature_count": profile.columns - 1,
+    }
+    return {
+        "feature_plan": feature_plan,
+        "trace": _trace(
+            state,
+            "Feature Engineering Agent",
+            started,
+            "Created leakage-safe ColumnTransformer plan.",
+        ),
+    }
+
+
+def modeling_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    retry = state.get("model_bundle").retry_number + 1 if state.get("model_bundle") else 0
+    bundle = train_models(
+        state["frame"],
+        state["target"],
+        state["profile"].task_type,
+        state["settings"],
+        retry_number=retry,
+    )
+    return {
+        "model_bundle": bundle,
+        "trace": _trace(
+            state,
+            "Modeling Agent",
+            started,
+            f"Compared {len(bundle.results)} models; {bundle.best_model} ranked first.",
+        ),
+    }
+
+
+def evaluation_critic_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    decision = critic_decision(state["model_bundle"], state["profile"].task_type, state["settings"])
+    detail = "Approved analysis." if decision.approved else "Rejected analysis and requested retry."
+    return {
+        "critic": decision,
+        "trace": _trace(state, "Evaluation / Critic Agent", started, detail),
+    }
+
+
+def critic_route(state: AgentState) -> str:
+    return "explainability" if state["critic"].approved else "retry_modeling"
+
+
+def explainability_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    result = explain_model(state["model_bundle"])
+    return {
+        "explainability": result,
+        "trace": _trace(
+            state, "Explainability Agent", started, f"Generated {result.method} explanations."
+        ),
+    }
+
+
+def executive_insights_agent(state: AgentState) -> dict[str, Any]:
+    started = time.perf_counter()
+    summary, recommendations = deterministic_insights(state)
+    llm_summary = optional_llm_narrative(state, state["evidence"], state["settings"])
+    if llm_summary:
+        summary = llm_summary
+    return {
+        "executive_summary": summary,
+        "recommendations": recommendations,
+        "trace": _trace(
+            state,
+            "Executive Insights Agent",
+            started,
+            "Created evidence-grounded narrative with deterministic metric provenance.",
+        ),
+    }
+
+
+def build_graph():
+    graph = StateGraph(AgentState)
+    graph.add_node("data_quality", data_quality_agent)
+    graph.add_node("eda", eda_agent)
+    graph.add_node("statistics", statistical_agent)
+    graph.add_node("planning", planning_agent)
+    graph.add_node("feature_engineering", feature_engineering_agent)
+    graph.add_node("modeling", modeling_agent)
+    graph.add_node("critic", evaluation_critic_agent)
+    graph.add_node("explainability", explainability_agent)
+    graph.add_node("executive_insights", executive_insights_agent)
+    graph.add_edge(START, "data_quality")
+    graph.add_edge("data_quality", "eda")
+    graph.add_edge("eda", "statistics")
+    graph.add_edge("statistics", "planning")
+    graph.add_edge("planning", "feature_engineering")
+    graph.add_edge("feature_engineering", "modeling")
+    graph.add_edge("modeling", "critic")
+    graph.add_conditional_edges(
+        "critic",
+        critic_route,
+        {"retry_modeling": "modeling", "explainability": "explainability"},
+    )
+    graph.add_edge("explainability", "executive_insights")
+    graph.add_edge("executive_insights", END)
+    return graph.compile()
+
+
+def run_analysis(
+    frame: pd.DataFrame,
+    target: str,
+    dataset_name: str,
+    settings: Settings | None = None,
+) -> RunSummary:
+    settings = settings or get_settings()
+    run_id = f"run_{uuid4().hex[:12]}"
+    store = RunStore(settings)
+    artifact_store = ArtifactStore(settings.artifact_root)
+    store.save(run_id, dataset_name, "running", {"run_id": run_id, "status": "running"})
+    initial: AgentState = {
+        "run_id": run_id,
+        "dataset_name": dataset_name,
+        "frame": frame,
+        "target": target,
+        "settings": settings,
+        "trace": [],
+    }
+    try:
+        final = build_graph().invoke(initial)
+        payload = _summary_payload(final, run_id, dataset_name)
+        final["summary_payload"] = payload
+        log_to_mlflow(payload, settings)
+        artifacts = export_artifacts(run_id, final, artifact_store.run_directory(run_id))
+        payload["artifacts"] = artifacts
+        payload["status"] = "completed"
+        store.save(run_id, dataset_name, "completed", payload)
+        return RunSummary.model_validate(payload)
+    except Exception as exc:
+        store.save(
+            run_id,
+            dataset_name,
+            "failed",
+            {"run_id": run_id, "dataset_name": dataset_name, "status": "failed", "error": str(exc)},
+        )
+        raise
+    finally:
+        store.engine.dispose()
+
+
+def _summary_payload(state: AgentState, run_id: str, dataset_name: str) -> dict[str, Any]:
+    bundle = state["model_bundle"]
+    return {
+        "run_id": run_id,
+        "status": "completed",
+        "dataset_name": dataset_name,
+        "profile": state["profile"].model_dump(mode="json"),
+        "plan": state["plan"].model_dump(mode="json"),
+        "quality_issues": [item.model_dump(mode="json") for item in state["quality_issues"]],
+        "evidence": [item.model_dump(mode="json") for item in state["evidence"]],
+        "model_results": [item.model_dump(mode="json") for item in bundle.results],
+        "model_failures": [item.model_dump(mode="json") for item in bundle.failures],
+        "best_model": bundle.best_model,
+        "critic": state["critic"].model_dump(mode="json"),
+        "explainability": state["explainability"].model_dump(mode="json"),
+        "executive_summary": state["executive_summary"],
+        "recommendations": state["recommendations"],
+        "artifacts": {},
+        "trace": state["trace"],
+    }

docker-compose.yml

@@ -0,0 +1,63 @@
+services:
+  ui:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - "8501:8501"
+    environment:
+      ARTIFACT_ROOT: /app/artifacts
+      DATABASE_URL: sqlite:////app/artifacts/datapilot.db
+    volumes:
+      - datapilot_artifacts:/app/artifacts
+    read_only: true
+    tmpfs:
+      - /tmp:size=128m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    mem_limit: 2g
+    cpus: 2
+
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile.api
+    ports:
+      - "8000:8000"
+    environment:
+      ARTIFACT_ROOT: /app/artifacts
+      DATABASE_URL: sqlite:////app/artifacts/datapilot.db
+      CORS_ORIGINS: http://localhost:8501
+    volumes:
+      - datapilot_artifacts:/app/artifacts
+    read_only: true
+    tmpfs:
+      - /tmp:size=128m
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    mem_limit: 2g
+    cpus: 2
+
+  worker:
+    build:
+      context: .
+      dockerfile: Dockerfile.worker
+    network_mode: none
+    read_only: true
+    tmpfs:
+      - /tmp:size=32m,noexec,nosuid
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    mem_limit: 256m
+    cpus: 0.5
+    pids_limit: 64
+
+volumes:
+  datapilot_artifacts:
+

docs/API_EXAMPLES.md

@@ -0,0 +1,37 @@
+# API examples
+
+Start the service with `uvicorn api.main:app --port 8000`. If `API_KEY` is configured, include
+`X-API-Key: <key>` in every `/v1/` request.
+
+## Submit a bundled dataset
+
+```bash
+curl -i -X POST http://localhost:8000/v1/analyze/sample \
+  -H "Content-Type: application/json" \
+  -H "Idempotency-Key: iris-demo-001" \
+  -d '{"sample":"iris"}'
+```
+
+```json
+{"job_id":"job_abc123","status":"queued"}
+```
+
+## Submit an upload
+
+```bash
+curl -i -X POST http://localhost:8000/v1/analyze/upload \
+  -F "file=@dataset.csv" \
+  -F "target=outcome"
+```
+
+## Poll or cancel
+
+```bash
+curl http://localhost:8000/v1/jobs/job_abc123
+curl -X DELETE http://localhost:8000/v1/jobs/job_abc123
+```
+
+A completed job includes the validated run summary in `result`. Errors exposed to clients are
+sanitized and carry an `X-Correlation-ID`; use that identifier to locate structured server logs.
+The machine-readable contract is committed as [`openapi.json`](openapi.json).
+

docs/ARCHITECTURE.md

@@ -0,0 +1,56 @@
+# DataPilot AI Architecture
+
+## Design goals
+
+DataPilot AI separates the user experience, orchestration, analytics, model training,
+persistence, and optional restricted computation. The default application remains useful
+without a paid model API; every displayed number originates from deterministic computation.
+
+## Runtime components
+
+| Component | Responsibility |
+|---|---|
+| Streamlit UI | Recruiter demo, upload/sample selection, charts, trace, downloads and Q&A |
+| FastAPI | Versioned analysis, run, artifact and evidence-backed Q&A endpoints |
+| LangGraph | Stateful agent ordering, state propagation and critic retry routing |
+| Analytics core | DuckDB profiling, statistical summaries and data-quality evidence |
+| ML core | Leakage-safe sklearn pipelines, model comparison and validation |
+| Explainability | SHAP when compatible; permutation-importance fallback |
+| Persistence | SQLAlchemy with SQLite locally and PostgreSQL via `DATABASE_URL` |
+| Artifact store | Local filesystem interface, replaceable by S3/MinIO |
+| Restricted worker | Expression-only AST validation in a networkless, resource-limited container |
+
+## Agent graph
+
+```mermaid
+flowchart TD
+    A["Dataset + target"] --> B["Data Quality Agent"]
+    B --> C["EDA Agent (DuckDB)"]
+    C --> D["Statistical Analysis Agent"]
+    D --> E["Planning Agent"]
+    E --> F["Feature Engineering Agent"]
+    F --> G["Modeling Agent"]
+    G --> H{"Evaluation / Critic Agent"}
+    H -->|"Reject: weak or unstable"| G
+    H -->|"Approve"| I["Explainability Agent"]
+    I --> J["Executive Insights Agent"]
+    J --> K["Report + model card + pipeline + evidence"]
+```
+
+## Leakage controls
+
+1. Rows without labels and exact duplicates are removed before splitting.
+2. Train/test split occurs before any learned transformation.
+3. Imputation, scaling, and one-hot encoding are inside `sklearn.pipeline.Pipeline`.
+4. Cross-validation refits the complete pipeline in every fold.
+5. Identifier and target-like features are flagged for human review.
+6. Holdout metrics and cross-validation metrics remain distinct.
+
+## Graceful degradation
+
+- Without Gemini: deterministic evidence-backed executive narrative.
+- Without SHAP: permutation importance.
+- Without XGBoost: sklearn candidate models.
+- Without MLflow: structured agent trace plus persisted run JSON.
+- Without PostgreSQL: SQLite.
+

docs/BENCHMARKS.md

@@ -0,0 +1,15 @@
+# Reproducible smoke benchmarks
+
+These results are correctness-oriented smoke benchmarks, not claims about production accuracy.
+They were measured on a local Windows CPU with Python 3.13.1, `RANDOM_STATE=42`,
+`OPTUNA_TRIALS=0`, and `MAX_CRITIC_RETRIES=0`.
+
+| Dataset | Rows | Task | Selected model | Training CV score | Untouched test score | Wall time |
+|---|---:|---|---|---:|---:|---:|
+| scikit-learn Iris | 150 | multiclass classification | Logistic Regression | 0.9583 weighted F1 | 0.9333 weighted F1 | 4.29 s |
+| scikit-learn Diabetes | 442 | regression | Linear Regression | 0.4493 R² | 0.4526 R² | 3.17 s |
+
+Candidate selection uses cross-validation on the training partition only. The test score is
+computed once after selection. Timings vary by hardware and installed optional dependencies.
+Run the same path with `python scripts/benchmark.py` once optional benchmark automation is added.
+

docs/DATA_PRIVACY.md

@@ -0,0 +1,10 @@
+# Data privacy
+
+DataPilot defaults to deterministic local analysis. Gemini is optional and receives a bounded
+metadata package rather than the full dataset: schema, aggregates, target candidates, quality
+signals, and a small redacted sample. Users can exclude columns, and likely PII columns are
+removed automatically.
+
+Production operators must define dataset retention, deletion SLAs, tenant isolation, regional
+processing, encryption, access logging, data-subject request handling, and subprocessors. API
+keys and database credentials belong in a secret manager and must never enter exports or logs.

docs/DEPLOYMENT.md

@@ -0,0 +1,37 @@
+# Deployment Guide
+
+## Streamlit Community Cloud
+
+1. Push the repository to GitHub.
+2. Create a Streamlit app with `streamlit_app.py` as the entrypoint.
+3. Use Python 3.12.
+4. Add `GEMINI_API_KEY` only if optional LLM narration is desired.
+5. Keep the default SQLite/local artifact mode for a stateless public demo.
+
+The app is fully functional without an LLM key.
+
+## Render API
+
+`render.yaml` defines a Docker-backed FastAPI service. Configure:
+
+- `DATABASE_URL`: Neon, Supabase or another PostgreSQL URL.
+- `GEMINI_API_KEY`: optional.
+- persistent/object storage if generated artifacts must survive redeployments.
+
+## Docker Compose
+
+```bash
+docker compose up --build
+```
+
+- Streamlit: <http://localhost:8501>
+- FastAPI: <http://localhost:8000/docs>
+
+The isolated worker has no exposed port and no network.
+
+## Production topology
+
+For heavier datasets, make `/v1/analyze/*` enqueue jobs to a separate worker and return
+`202 Accepted`. Persist state in PostgreSQL, place artifacts in S3-compatible storage, and
+stream progress through server-sent events or polling.
+