Update audited Customer Churn models and app
Browse files- .dockerignore +9 -0
- .github/workflows/ci.yml +23 -0
- .gitignore +8 -0
- Dockerfile +12 -0
- MODEL_CARD.md +27 -0
- README.md +74 -6
- app.py +194 -0
- model_metadata.json +415 -0
- models/bank_churn.joblib +3 -0
- models/telecom_churn.joblib +3 -0
- notebooks/Customer_Churn_Training_Walkthrough.ipynb +129 -0
- reports/bank_confusion_matrix.png +0 -0
- reports/bank_feature_importance.csv +10 -0
- reports/bank_feature_importance.png +0 -0
- reports/bank_roc.png +0 -0
- reports/model_comparison.csv +7 -0
- reports/telecom_confusion_matrix.png +0 -0
- reports/telecom_feature_importance.csv +11 -0
- reports/telecom_feature_importance.png +0 -0
- reports/telecom_roc.png +0 -0
- requirements.txt +9 -0
- tests/test_api.py +63 -0
- train.py +271 -0
.dockerignore
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__
|
| 2 |
+
.ipynb_checkpoints
|
| 3 |
+
*.pyc
|
| 4 |
+
.git
|
| 5 |
+
churn_inputs/
|
| 6 |
+
bank_customer_churn*.csv
|
| 7 |
+
customer_churn_data*.csv
|
| 8 |
+
.env
|
| 9 |
+
**/.env
|
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI — Model and API smoke tests
|
| 2 |
+
on:
|
| 3 |
+
push:
|
| 4 |
+
pull_request:
|
| 5 |
+
jobs:
|
| 6 |
+
test:
|
| 7 |
+
runs-on: ubuntu-latest
|
| 8 |
+
steps:
|
| 9 |
+
- uses: actions/checkout@v4
|
| 10 |
+
- uses: actions/setup-python@v5
|
| 11 |
+
with:
|
| 12 |
+
python-version: '3.13'
|
| 13 |
+
- name: Install dependencies
|
| 14 |
+
run: |
|
| 15 |
+
python -m pip install --upgrade pip
|
| 16 |
+
pip install -r requirements.txt 'pytest>=8,<10' 'httpx>=0.27,<1'
|
| 17 |
+
- name: Validate stored model versions
|
| 18 |
+
run: |
|
| 19 |
+
python -c "import json,sklearn,xgboost; m=json.load(open('model_metadata.json')); assert m['sklearn_version']==sklearn.__version__; assert m['xgboost_version']==xgboost.__version__"
|
| 20 |
+
- name: Compile and test REST API
|
| 21 |
+
run: |
|
| 22 |
+
python -m py_compile app.py train.py
|
| 23 |
+
python -m pytest -q tests
|
.gitignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.ipynb_checkpoints/
|
| 4 |
+
.env
|
| 5 |
+
*.csv
|
| 6 |
+
!reports/*.csv
|
| 7 |
+
churn_inputs/
|
| 8 |
+
*.zip
|
Dockerfile
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.13-slim
|
| 2 |
+
RUN apt-get update && apt-get install -y --no-install-recommends libgomp1 \
|
| 3 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 4 |
+
RUN useradd -m -u 1000 user
|
| 5 |
+
USER user
|
| 6 |
+
ENV HOME=/home/user PATH=/home/user/.local/bin:$PATH PYTHONUNBUFFERED=1
|
| 7 |
+
WORKDIR /home/user/app
|
| 8 |
+
COPY --chown=user requirements.txt ./requirements.txt
|
| 9 |
+
RUN pip install --no-cache-dir --user -r requirements.txt
|
| 10 |
+
COPY --chown=user . .
|
| 11 |
+
EXPOSE 7860
|
| 12 |
+
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
|
MODEL_CARD.md
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Model Card: Bank + Telecom Churn Screening
|
| 2 |
+
|
| 3 |
+
## Bank
|
| 4 |
+
Predictors: CreditScore, Age, Tenure, Balance, NumOfProducts, HasCrCard, IsActiveMember, EstimatedSalary, Geography.
|
| 5 |
+
|
| 6 |
+
Excluded identifiers: CustomerId, Surname.
|
| 7 |
+
|
| 8 |
+
Duplicate rows removed: 2. Train/validation/test: 6400/1600/2000.
|
| 9 |
+
|
| 10 |
+
Model: XGBoost; validation-selected F2 cutoff: 0.425; quality gate: demonstration.
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
## Telecom
|
| 14 |
+
Predictors: SeniorCitizen, tenure, MonthlyCharges, TotalCharges, Partner, Dependents, PhoneService, InternetService, Contract, PaymentMethod.
|
| 15 |
+
|
| 16 |
+
Excluded identifiers: customerID.
|
| 17 |
+
|
| 18 |
+
Duplicate rows removed: 0. Train/validation/test: 3763/941/1176.
|
| 19 |
+
|
| 20 |
+
Model: XGBoost; validation-selected F2 cutoff: 0.100; quality gate: research_only_low_signal.
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
## Methodology
|
| 24 |
+
Training-only preprocessing, validation PR AUC model selection, validation F2 threshold selection, untouched test performance.
|
| 25 |
+
|
| 26 |
+
## Limitations
|
| 27 |
+
Educational datasets, distribution shift, no probability calibration, no subgroup fairness validation, and no downstream business impact study. For demonstration only.
|
README.md
CHANGED
|
@@ -1,12 +1,80 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: docker
|
|
|
|
| 7 |
pinned: false
|
| 8 |
-
license: other
|
| 9 |
-
short_description: Visitors will enter customer data, run the deployed model
|
| 10 |
---
|
| 11 |
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Churn Intelligence Lab
|
| 3 |
+
emoji: 📊
|
| 4 |
+
colorFrom: blue
|
| 5 |
colorTo: indigo
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
pinned: false
|
|
|
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
+
# Churn Intelligence Lab — dual-industry ML portfolio
|
| 12 |
+
|
| 13 |
+
**Live demo**: A Docker-hosted Gradio + FastAPI application with two separately trained models.
|
| 14 |
+
**How it works**: Bank and telecom customer profiles → in-pipeline imputation/encoding → classifier → thresholded churn screening score.
|
| 15 |
+
|
| 16 |
+
## Evaluated results (held-out 20% test partition)
|
| 17 |
+
|
| 18 |
+
| Dataset | Records after dedup | Selected model | ROC AUC | PR AUC | Precision | Recall | F2 | Threshold | Status |
|
| 19 |
+
|---|---:|---|---:|---:|---:|---:|---:|---:|---|
|
| 20 |
+
| Bank | 10,000 | XGBoost | 0.858 | 0.703 | 0.440 | 0.813 | 0.695 | 0.425 | demonstration |
|
| 21 |
+
| Telecom | 5,880 | XGBoost | 0.480 | 0.471 | 0.493 | 1.000 | 0.830 | 0.100 | research_only_low_signal |
|
| 22 |
+
|
| 23 |
+
**Evaluation protocol:** 64/16/20 train/validation/test stratified splits, fixed random seed, model selection on validation PR AUC; the F2 classification threshold is selected on validation only. The test set remains untouched until final scoring. XGBoost uses training-only 3-fold randomized hyperparameter search.
|
| 24 |
+
|
| 25 |
+
## Architecture
|
| 26 |
+
|
| 27 |
+
`train.py` → `models/{bank,telecom}_churn.joblib` + `model_metadata.json` → `app.py` → Docker Space (port 7860) → Gradio UI at `/ui/` + FastAPI Swagger at `/docs`.
|
| 28 |
+
|
| 29 |
+
### Public API
|
| 30 |
+
- `GET /health`: readiness and loaded models
|
| 31 |
+
- `GET /api/metadata`: columns, chosen model, test metrics
|
| 32 |
+
- `POST /api/predict/bank` or `/api/predict/telecom`: `{"data": {...feature name/value pairs...}}`
|
| 33 |
+
- `POST /api/predict-batch/bank` or `/api/predict-batch/telecom`: `{"records": [{...}, {...}]}` (max 100)
|
| 34 |
+
|
| 35 |
+
**Sample request** (generated safely from defaults in model metadata):
|
| 36 |
+
```python
|
| 37 |
+
import json, requests
|
| 38 |
+
metadata = json.load(open('model_metadata.json'))['datasets']['bank']
|
| 39 |
+
payload = {'data': metadata['defaults']}
|
| 40 |
+
response = requests.post('https://YOUR-SPACE.hf.space/api/predict/bank', json=payload, timeout=20)
|
| 41 |
+
print(response.json())
|
| 42 |
+
```
|
| 43 |
+
|
| 44 |
+
## Reproduce model training
|
| 45 |
+
1. Use the Python minor version in the notebook-generated `Dockerfile` (and `model_metadata.json` when published). Install: `pip install -r requirements.txt matplotlib`.
|
| 46 |
+
2. Supply `bank_customer_churn.csv` and `customer_churn_data.csv` locally, **never publish raw customer rows**.
|
| 47 |
+
3. `python train.py --data-dir /path/to/datasets --project-dir . --tuning-iterations 6`.
|
| 48 |
+
4. `python -m pytest -q tests/`; `uvicorn app:app --host 0.0.0.0 --port 7860`.
|
| 49 |
+
5. Visit `http://localhost:7860/ui/` and `http://localhost:7860/docs`.
|
| 50 |
+
|
| 51 |
+
## Evaluation visualizations
|
| 52 |
+
|
| 53 |
+
**Bank ROC (held-out test)**
|
| 54 |
+
|
| 55 |
+

|
| 56 |
+
|
| 57 |
+
**Bank feature importance (validation permutation)**
|
| 58 |
+
|
| 59 |
+

|
| 60 |
+
|
| 61 |
+
**Telecom ROC (near-chance baseline)**
|
| 62 |
+
|
| 63 |
+

|
| 64 |
+
|
| 65 |
+
## Project layout
|
| 66 |
+
- `train.py` — full reusable training and model comparison pipeline
|
| 67 |
+
- `app.py` — UI + validated REST endpoints
|
| 68 |
+
- `models/` — two **trained**, serialized sklearn-compatible pipelines
|
| 69 |
+
- `reports/` — comparison CSV, permutation importance, ROC and confusion matrix graphics
|
| 70 |
+
- `model_metadata.json` — measured performance, feature defaults, audits
|
| 71 |
+
- `tests/` — API smoke tests
|
| 72 |
+
- `Dockerfile`, `requirements.txt` — reproducible runtime
|
| 73 |
+
- `MODEL_CARD.md` — model/data limitations and ethics notes
|
| 74 |
+
- `notebooks/Customer_Churn_Training_Walkthrough.ipynb` — standalone training walkthrough runnable in Colab
|
| 75 |
+
- `.github/workflows/ci.yml` — GitHub Actions CI smoke test
|
| 76 |
+
|
| 77 |
+
## Limitations and responsible use
|
| 78 |
+
Scores are **uncalibrated** probabilities and are not guarantees. A model with near-chance held-out discrimination is flagged `research_only_low_signal`, and its API refuses actionable churn labels. Demonstration datasets may be artificial/educational and do not prove real-world performance; monitor data drift, assess geographic/demographic fairness, calibrate before operational use, and obtain appropriate consent and privacy reviews. The public demo is for **synthetic or anonymized inputs**, with no storage of uploaded customer records as a product feature. Do not submit personal data or use this demo to make automated decisions about people.
|
| 79 |
+
|
| 80 |
+
*Built in a single Colab notebook from the attached deployment and containerization case studies. All reported results are computed, not estimated.*
|
app.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Docker Space: FastAPI REST service plus Gradio web UI, powered by trained pipelines."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import json
|
| 4 |
+
import tempfile
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
|
| 7 |
+
import gradio as gr
|
| 8 |
+
import joblib
|
| 9 |
+
import numpy as np
|
| 10 |
+
import pandas as pd
|
| 11 |
+
from fastapi import FastAPI, HTTPException
|
| 12 |
+
from fastapi.responses import RedirectResponse
|
| 13 |
+
from pydantic import BaseModel, Field
|
| 14 |
+
|
| 15 |
+
ROOT = Path(__file__).resolve().parent
|
| 16 |
+
META = json.loads((ROOT / 'model_metadata.json').read_text(encoding='utf-8'))['datasets']
|
| 17 |
+
MODELS = {key: joblib.load(ROOT / 'models' / f'{key}_churn.joblib') for key in META}
|
| 18 |
+
|
| 19 |
+
class SinglePayload(BaseModel):
|
| 20 |
+
data: dict
|
| 21 |
+
|
| 22 |
+
class BatchPayload(BaseModel):
|
| 23 |
+
records: list[dict] = Field(min_length=1, max_length=100)
|
| 24 |
+
|
| 25 |
+
def validate_input(key, records):
|
| 26 |
+
if key not in MODELS:
|
| 27 |
+
raise ValueError(f'Unknown segment {key!r}; choose {list(MODELS)}')
|
| 28 |
+
if not records or len(records) > 100:
|
| 29 |
+
raise ValueError('Provide 1–100 records per request')
|
| 30 |
+
info = META[key]
|
| 31 |
+
required = info['features']
|
| 32 |
+
X = pd.DataFrame(records)
|
| 33 |
+
absent = [c for c in required if c not in X.columns]
|
| 34 |
+
if absent:
|
| 35 |
+
raise ValueError(f'Missing required features for {key}: {absent}')
|
| 36 |
+
X = X[required].copy() # Ignore arbitrary PII/extraneous uploaded columns.
|
| 37 |
+
for c in info['numeric']:
|
| 38 |
+
vals = pd.to_numeric(X[c], errors='coerce')
|
| 39 |
+
if vals.isna().any() or not np.isfinite(vals.to_numpy(dtype=float)).all():
|
| 40 |
+
raise ValueError(f'{c} must contain valid finite numeric values; empty inputs are not accepted')
|
| 41 |
+
X[c] = vals
|
| 42 |
+
for c in info['categorical']:
|
| 43 |
+
allowed = info['categorical_choices'][c]
|
| 44 |
+
vals = X[c].astype('string').str.strip()
|
| 45 |
+
if vals.isna().any() or (~vals.isin(allowed)).any():
|
| 46 |
+
raise ValueError(f'{c} must be one of {allowed}')
|
| 47 |
+
X[c] = vals.astype(str)
|
| 48 |
+
return X
|
| 49 |
+
|
| 50 |
+
def predict_records(key, records):
|
| 51 |
+
frame = validate_input(key, records)
|
| 52 |
+
probs = MODELS[key]['pipeline'].predict_proba(frame)[:, 1]
|
| 53 |
+
cut = float(MODELS[key]['threshold'])
|
| 54 |
+
low_signal = META[key]['deployment_status'] != 'demonstration'
|
| 55 |
+
return [{'row_number': i+1, 'churn_score': round(float(p), 5),
|
| 56 |
+
'prediction': ('Exploratory only — unreliable classifier' if low_signal
|
| 57 |
+
else ('Likely to churn' if p >= cut else 'Likely to stay')),
|
| 58 |
+
'deployment_status': META[key]['deployment_status'],
|
| 59 |
+
'threshold': cut} for i, p in enumerate(probs)]
|
| 60 |
+
|
| 61 |
+
api = FastAPI(title='Churn Intelligence Lab API', version='1.0.0',
|
| 62 |
+
description='Demo-only bank and telecom churn inference. Do not submit real customer PII.')
|
| 63 |
+
|
| 64 |
+
@api.get('/', include_in_schema=False)
|
| 65 |
+
def home():
|
| 66 |
+
return RedirectResponse('/ui/')
|
| 67 |
+
|
| 68 |
+
@api.get('/health')
|
| 69 |
+
def health():
|
| 70 |
+
return {'status': 'ok', 'trained_models': list(MODELS), 'version': '1.0.0'}
|
| 71 |
+
|
| 72 |
+
@api.get('/api/metadata')
|
| 73 |
+
def metadata():
|
| 74 |
+
return {name: {'features': info['features'], 'threshold': info['threshold'],
|
| 75 |
+
'selected_model': info['selected_model'], 'test_metrics': info['test_metrics']}
|
| 76 |
+
for name, info in META.items()}
|
| 77 |
+
|
| 78 |
+
@api.post('/api/predict/{segment}')
|
| 79 |
+
def predict_one(segment: str, body: SinglePayload):
|
| 80 |
+
try:
|
| 81 |
+
return predict_records(segment, [body.data])[0]
|
| 82 |
+
except ValueError as error:
|
| 83 |
+
raise HTTPException(status_code=422, detail=str(error)) from error
|
| 84 |
+
|
| 85 |
+
@api.post('/api/predict-batch/{segment}')
|
| 86 |
+
def predict_many(segment: str, body: BatchPayload):
|
| 87 |
+
try:
|
| 88 |
+
return {'results': predict_records(segment, body.records)}
|
| 89 |
+
except ValueError as error:
|
| 90 |
+
raise HTTPException(status_code=422, detail=str(error)) from error
|
| 91 |
+
|
| 92 |
+
def format_prediction(segment, *values):
|
| 93 |
+
fields = META[segment]['features']
|
| 94 |
+
response = predict_records(segment, [dict(zip(fields, values))])[0]
|
| 95 |
+
return (f'### {response["prediction"]}\n\n'
|
| 96 |
+
f'**Churn model score:** {response["churn_score"]:.1%} '\
|
| 97 |
+
f'\n\n**Decision threshold:** {response["threshold"]:.1%} '
|
| 98 |
+
f'\n\n*Uncalibrated model output; screening aid, not a guarantee of churn.*'), response
|
| 99 |
+
|
| 100 |
+
def run_batch(segment, filepath):
|
| 101 |
+
if not filepath:
|
| 102 |
+
raise gr.Error('Upload a CSV file before requesting batch predictions.')
|
| 103 |
+
try:
|
| 104 |
+
raw = pd.read_csv(filepath)
|
| 105 |
+
if len(raw) == 0 or len(raw) > 100:
|
| 106 |
+
raise ValueError('CSV must contain between 1 and 100 rows')
|
| 107 |
+
output = predict_records(segment, raw.to_dict(orient='records'))
|
| 108 |
+
result_path = Path(tempfile.mkdtemp(prefix='churn_demo_')) / f'{segment}_prediction_results.csv'
|
| 109 |
+
pd.DataFrame(output).to_csv(result_path, index=False)
|
| 110 |
+
return pd.DataFrame(output), str(result_path)
|
| 111 |
+
except (ValueError, pd.errors.ParserError) as error:
|
| 112 |
+
raise gr.Error(str(error)) from error
|
| 113 |
+
|
| 114 |
+
SITE_CSS = '''
|
| 115 |
+
.gradio-container {max-width: 1240px !important; margin: auto !important;}
|
| 116 |
+
#hero {background:linear-gradient(120deg,#0a2342,#123a68 65%,#176a89);border-radius:20px;
|
| 117 |
+
padding:28px 34px;color:white; box-shadow:0 12px 40px rgba(5,22,45,.15);}
|
| 118 |
+
#hero h1,#hero p {color:white !important;}
|
| 119 |
+
#hero h1 {font-weight:800;letter-spacing:-.03em;font-size:2.2rem;}
|
| 120 |
+
#metricline {padding:10px 16px;border:1px solid #dce6ef;border-radius:12px;}
|
| 121 |
+
#caution {background:#fff7ed;border:1px solid #fed7aa;padding:12px 16px;border-radius:12px;}
|
| 122 |
+
#resultbox {border-left:4px solid #0e7490;padding:8px 16px;min-height:90px;}
|
| 123 |
+
'''
|
| 124 |
+
|
| 125 |
+
with gr.Blocks(title='Churn Intelligence Lab | AI Customer Retention', css=SITE_CSS,
|
| 126 |
+
theme=gr.themes.Soft(primary_hue='blue', neutral_hue='slate')) as demo:
|
| 127 |
+
gr.Markdown('''<div id="hero"><p style="opacity:.8;font-weight:700;letter-spacing:.13em">AI • MACHINE LEARNING • MLOPS</p>
|
| 128 |
+
<h1>Churn Intelligence Lab</h1>
|
| 129 |
+
<p>Explore AI-assisted customer attrition insights across banking and telecom. Compare model evidence, run individual assessments, and export anonymized batch results.</p>
|
| 130 |
+
<p style="font-size:.86em;opacity:.85">Portfolio research demonstration • Do not enter real customer identifiers or sensitive information.</p></div>''')
|
| 131 |
+
with gr.Row():
|
| 132 |
+
gr.Markdown('**Two independently evaluated datasets** \nTrain/test separation · Reproducible preprocessing · Saved production-style pipelines')
|
| 133 |
+
gr.Markdown('**Developer access** \n[REST API docs](/docs) · [Model metadata](/api/metadata) · [Service health](/health)')
|
| 134 |
+
for key, info in META.items():
|
| 135 |
+
with gr.Tab('🏦 Banking attrition' if key == 'bank' else '📶 Telecom attrition'):
|
| 136 |
+
met = info['test_metrics']
|
| 137 |
+
with gr.Row():
|
| 138 |
+
gr.Markdown(f'''### {"Banking" if key == "bank" else "Telecommunications"} model
|
| 139 |
+
<div id="metricline"> <b>Selected:</b> {info["selected_model"]} |
|
| 140 |
+
<b>Test ROC AUC:</b> {met["roc_auc"]:.3f} |
|
| 141 |
+
<b>PR AUC:</b> {met["pr_auc"]:.3f} |
|
| 142 |
+
<b>Recall:</b> {met["recall"]:.3f}</div>''')
|
| 143 |
+
if info['deployment_status'] != 'demonstration':
|
| 144 |
+
gr.Markdown('''<div id="caution"><b>⚠ Experimental — low predictive reliability.</b>
|
| 145 |
+
The held-out evaluation is near chance. Scores are exploratory; this model is not suitable for business churn decisions.</div>''')
|
| 146 |
+
else:
|
| 147 |
+
gr.Markdown('**Customer assessment** — review sample defaults, modify customer attributes, and run a prediction.')
|
| 148 |
+
inputs=[]
|
| 149 |
+
with gr.Accordion('Customer profile inputs', open=True):
|
| 150 |
+
with gr.Row():
|
| 151 |
+
with gr.Column():
|
| 152 |
+
first=info['features'][::2]
|
| 153 |
+
for col in first:
|
| 154 |
+
if col in info['numeric']:
|
| 155 |
+
component=gr.Number(label=col, value=info['defaults'][col], precision=2)
|
| 156 |
+
else:
|
| 157 |
+
component=gr.Dropdown(label=col, choices=info['categorical_choices'][col],value=info['defaults'][col])
|
| 158 |
+
inputs.append((col,component))
|
| 159 |
+
with gr.Column():
|
| 160 |
+
for col in info['features'][1::2]:
|
| 161 |
+
if col in info['numeric']:
|
| 162 |
+
component=gr.Number(label=col, value=info['defaults'][col], precision=2)
|
| 163 |
+
else:
|
| 164 |
+
component=gr.Dropdown(label=col, choices=info['categorical_choices'][col],value=info['defaults'][col])
|
| 165 |
+
inputs.append((col,component))
|
| 166 |
+
ordered=[dict(inputs)[col] for col in info['features']]
|
| 167 |
+
run=gr.Button('Analyze customer churn risk',variant='primary',size='lg')
|
| 168 |
+
with gr.Row():
|
| 169 |
+
with gr.Column(scale=2):
|
| 170 |
+
textout=gr.Markdown('Ready to analyze a synthetic customer profile.',elem_id='resultbox')
|
| 171 |
+
with gr.Column(scale=1):
|
| 172 |
+
jsonout=gr.JSON(label='Prediction details')
|
| 173 |
+
run.click(fn=lambda *values,segment=key: format_prediction(segment,*values),
|
| 174 |
+
inputs=ordered,outputs=[textout,jsonout],api_name=False)
|
| 175 |
+
gr.Markdown('### Batch screening (CSV)\nUpload 1–100 rows with the required feature columns. Only anonymized or synthetic data is permitted.')
|
| 176 |
+
with gr.Row():
|
| 177 |
+
csvfile=gr.File(label='Anonymized feature CSV',file_types=['.csv'],type='filepath')
|
| 178 |
+
batch_button=gr.Button('Process CSV batch')
|
| 179 |
+
with gr.Row():
|
| 180 |
+
batch_table=gr.Dataframe(label='Prediction results',interactive=False)
|
| 181 |
+
batch_download=gr.File(label='Download results CSV')
|
| 182 |
+
batch_button.click(fn=lambda f,segment=key: run_batch(segment,f),
|
| 183 |
+
inputs=csvfile,outputs=[batch_table,batch_download],api_name=False)
|
| 184 |
+
gr.Markdown('''---
|
| 185 |
+
**How to interpret scores:** estimated model probabilities are not calibrated. Thresholds are validation-derived; outcomes are uncertain.
|
| 186 |
+
Telecom is research-only because of weak signal. Evaluate fairness, drift, and real-world fit before any operational use.
|
| 187 |
+
|
| 188 |
+
**Architecture:** Python · Scikit-learn · XGBoost · Gradio · FastAPI · Docker · Hugging Face Spaces''')
|
| 189 |
+
|
| 190 |
+
app = gr.mount_gradio_app(api, demo, path='/ui')
|
| 191 |
+
|
| 192 |
+
if __name__ == '__main__':
|
| 193 |
+
import uvicorn
|
| 194 |
+
uvicorn.run(app, host='0.0.0.0', port=7860)
|
model_metadata.json
ADDED
|
@@ -0,0 +1,415 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"generated_utc": "2026-10-17T21:15:10.673962+00:00",
|
| 3 |
+
"random_seed": 42,
|
| 4 |
+
"sklearn_version": "1.8.0",
|
| 5 |
+
"xgboost_version": "3.1.3",
|
| 6 |
+
"python_version": "3.13.16",
|
| 7 |
+
"datasets": {
|
| 8 |
+
"bank": {
|
| 9 |
+
"label": "Bank customer churn",
|
| 10 |
+
"selected_model": "XGBoost",
|
| 11 |
+
"numeric": [
|
| 12 |
+
"CreditScore",
|
| 13 |
+
"Age",
|
| 14 |
+
"Tenure",
|
| 15 |
+
"Balance",
|
| 16 |
+
"NumOfProducts",
|
| 17 |
+
"HasCrCard",
|
| 18 |
+
"IsActiveMember",
|
| 19 |
+
"EstimatedSalary"
|
| 20 |
+
],
|
| 21 |
+
"categorical": [
|
| 22 |
+
"Geography"
|
| 23 |
+
],
|
| 24 |
+
"features": [
|
| 25 |
+
"CreditScore",
|
| 26 |
+
"Age",
|
| 27 |
+
"Tenure",
|
| 28 |
+
"Balance",
|
| 29 |
+
"NumOfProducts",
|
| 30 |
+
"HasCrCard",
|
| 31 |
+
"IsActiveMember",
|
| 32 |
+
"EstimatedSalary",
|
| 33 |
+
"Geography"
|
| 34 |
+
],
|
| 35 |
+
"excluded_columns": [
|
| 36 |
+
"CustomerId",
|
| 37 |
+
"Surname"
|
| 38 |
+
],
|
| 39 |
+
"categorical_choices": {
|
| 40 |
+
"Geography": [
|
| 41 |
+
"France",
|
| 42 |
+
"Germany",
|
| 43 |
+
"Spain"
|
| 44 |
+
]
|
| 45 |
+
},
|
| 46 |
+
"defaults": {
|
| 47 |
+
"CreditScore": 653.0,
|
| 48 |
+
"Age": 37.0,
|
| 49 |
+
"Tenure": 5.0,
|
| 50 |
+
"Balance": 97452.79500000001,
|
| 51 |
+
"NumOfProducts": 1.0,
|
| 52 |
+
"HasCrCard": 1.0,
|
| 53 |
+
"IsActiveMember": 1.0,
|
| 54 |
+
"EstimatedSalary": 99482.98,
|
| 55 |
+
"Geography": "France"
|
| 56 |
+
},
|
| 57 |
+
"threshold": 0.425,
|
| 58 |
+
"training_rows": 6400,
|
| 59 |
+
"validation_rows": 1600,
|
| 60 |
+
"test_rows": 2000,
|
| 61 |
+
"audit": {
|
| 62 |
+
"original_rows": 10002,
|
| 63 |
+
"duplicate_rows_removed": 2,
|
| 64 |
+
"rows_after_deduplication": 10000,
|
| 65 |
+
"positive_rate": 0.2037,
|
| 66 |
+
"feature_missing_counts": {
|
| 67 |
+
"CreditScore": 0,
|
| 68 |
+
"Age": 1,
|
| 69 |
+
"Tenure": 0,
|
| 70 |
+
"Balance": 0,
|
| 71 |
+
"NumOfProducts": 0,
|
| 72 |
+
"HasCrCard": 1,
|
| 73 |
+
"IsActiveMember": 1,
|
| 74 |
+
"EstimatedSalary": 0,
|
| 75 |
+
"Geography": 1
|
| 76 |
+
},
|
| 77 |
+
"sha256_first12": "8c8d77064a1c"
|
| 78 |
+
},
|
| 79 |
+
"validation_metrics": {
|
| 80 |
+
"roc_auc": 0.85564,
|
| 81 |
+
"pr_auc": 0.68442,
|
| 82 |
+
"balanced_accuracy": 0.77889,
|
| 83 |
+
"precision": 0.44915,
|
| 84 |
+
"recall": 0.81288,
|
| 85 |
+
"f1": 0.5786,
|
| 86 |
+
"f2": 0.69958,
|
| 87 |
+
"confusion_matrix": [
|
| 88 |
+
[
|
| 89 |
+
949,
|
| 90 |
+
325
|
| 91 |
+
],
|
| 92 |
+
[
|
| 93 |
+
61,
|
| 94 |
+
265
|
| 95 |
+
]
|
| 96 |
+
],
|
| 97 |
+
"classification_report": {
|
| 98 |
+
"0": {
|
| 99 |
+
"precision": 0.9396039603960396,
|
| 100 |
+
"recall": 0.7448979591836735,
|
| 101 |
+
"f1-score": 0.8309982486865148,
|
| 102 |
+
"support": 1274.0
|
| 103 |
+
},
|
| 104 |
+
"1": {
|
| 105 |
+
"precision": 0.4491525423728814,
|
| 106 |
+
"recall": 0.8128834355828221,
|
| 107 |
+
"f1-score": 0.5786026200873362,
|
| 108 |
+
"support": 326.0
|
| 109 |
+
},
|
| 110 |
+
"accuracy": 0.75875,
|
| 111 |
+
"macro avg": {
|
| 112 |
+
"precision": 0.6943782513844605,
|
| 113 |
+
"recall": 0.7788906973832478,
|
| 114 |
+
"f1-score": 0.7048004343869255,
|
| 115 |
+
"support": 1600.0
|
| 116 |
+
},
|
| 117 |
+
"weighted avg": {
|
| 118 |
+
"precision": 0.8396744839738212,
|
| 119 |
+
"recall": 0.75875,
|
| 120 |
+
"f1-score": 0.7795726393594322,
|
| 121 |
+
"support": 1600.0
|
| 122 |
+
}
|
| 123 |
+
}
|
| 124 |
+
},
|
| 125 |
+
"test_metrics": {
|
| 126 |
+
"roc_auc": 0.85784,
|
| 127 |
+
"pr_auc": 0.70287,
|
| 128 |
+
"balanced_accuracy": 0.77449,
|
| 129 |
+
"precision": 0.44016,
|
| 130 |
+
"recall": 0.81327,
|
| 131 |
+
"f1": 0.57118,
|
| 132 |
+
"f2": 0.69538,
|
| 133 |
+
"confusion_matrix": [
|
| 134 |
+
[
|
| 135 |
+
1172,
|
| 136 |
+
421
|
| 137 |
+
],
|
| 138 |
+
[
|
| 139 |
+
76,
|
| 140 |
+
331
|
| 141 |
+
]
|
| 142 |
+
],
|
| 143 |
+
"classification_report": {
|
| 144 |
+
"0": {
|
| 145 |
+
"precision": 0.9391025641025641,
|
| 146 |
+
"recall": 0.7357187696170747,
|
| 147 |
+
"f1-score": 0.825061598028863,
|
| 148 |
+
"support": 1593.0
|
| 149 |
+
},
|
| 150 |
+
"1": {
|
| 151 |
+
"precision": 0.4401595744680851,
|
| 152 |
+
"recall": 0.8132678132678133,
|
| 153 |
+
"f1-score": 0.5711820534943917,
|
| 154 |
+
"support": 407.0
|
| 155 |
+
},
|
| 156 |
+
"accuracy": 0.7515,
|
| 157 |
+
"macro avg": {
|
| 158 |
+
"precision": 0.6896310692853246,
|
| 159 |
+
"recall": 0.774493291442444,
|
| 160 |
+
"f1-score": 0.6981218257616273,
|
| 161 |
+
"support": 2000.0
|
| 162 |
+
},
|
| 163 |
+
"weighted avg": {
|
| 164 |
+
"precision": 0.8375676657119476,
|
| 165 |
+
"recall": 0.7515,
|
| 166 |
+
"f1-score": 0.7733971107160982,
|
| 167 |
+
"support": 2000.0
|
| 168 |
+
}
|
| 169 |
+
}
|
| 170 |
+
},
|
| 171 |
+
"deployment_status": "demonstration",
|
| 172 |
+
"model_size_mb": 0.037,
|
| 173 |
+
"validation_model_comparison": [
|
| 174 |
+
{
|
| 175 |
+
"dataset": "bank",
|
| 176 |
+
"model": "XGBoost",
|
| 177 |
+
"validation_pr_auc": 0.68442,
|
| 178 |
+
"validation_roc_auc": 0.85564,
|
| 179 |
+
"tuning": "6x3 CV"
|
| 180 |
+
},
|
| 181 |
+
{
|
| 182 |
+
"dataset": "bank",
|
| 183 |
+
"model": "Random forest",
|
| 184 |
+
"validation_pr_auc": 0.68408,
|
| 185 |
+
"validation_roc_auc": 0.86424,
|
| 186 |
+
"tuning": "none"
|
| 187 |
+
},
|
| 188 |
+
{
|
| 189 |
+
"dataset": "bank",
|
| 190 |
+
"model": "Logistic regression",
|
| 191 |
+
"validation_pr_auc": 0.45514,
|
| 192 |
+
"validation_roc_auc": 0.75422,
|
| 193 |
+
"tuning": "none"
|
| 194 |
+
}
|
| 195 |
+
]
|
| 196 |
+
},
|
| 197 |
+
"telecom": {
|
| 198 |
+
"label": "Telecom customer churn",
|
| 199 |
+
"selected_model": "XGBoost",
|
| 200 |
+
"numeric": [
|
| 201 |
+
"SeniorCitizen",
|
| 202 |
+
"tenure",
|
| 203 |
+
"MonthlyCharges",
|
| 204 |
+
"TotalCharges"
|
| 205 |
+
],
|
| 206 |
+
"categorical": [
|
| 207 |
+
"Partner",
|
| 208 |
+
"Dependents",
|
| 209 |
+
"PhoneService",
|
| 210 |
+
"InternetService",
|
| 211 |
+
"Contract",
|
| 212 |
+
"PaymentMethod"
|
| 213 |
+
],
|
| 214 |
+
"features": [
|
| 215 |
+
"SeniorCitizen",
|
| 216 |
+
"tenure",
|
| 217 |
+
"MonthlyCharges",
|
| 218 |
+
"TotalCharges",
|
| 219 |
+
"Partner",
|
| 220 |
+
"Dependents",
|
| 221 |
+
"PhoneService",
|
| 222 |
+
"InternetService",
|
| 223 |
+
"Contract",
|
| 224 |
+
"PaymentMethod"
|
| 225 |
+
],
|
| 226 |
+
"excluded_columns": [
|
| 227 |
+
"customerID"
|
| 228 |
+
],
|
| 229 |
+
"categorical_choices": {
|
| 230 |
+
"Partner": [
|
| 231 |
+
"No",
|
| 232 |
+
"Yes"
|
| 233 |
+
],
|
| 234 |
+
"Dependents": [
|
| 235 |
+
"No",
|
| 236 |
+
"Yes"
|
| 237 |
+
],
|
| 238 |
+
"PhoneService": [
|
| 239 |
+
"No",
|
| 240 |
+
"Yes"
|
| 241 |
+
],
|
| 242 |
+
"InternetService": [
|
| 243 |
+
"DSL",
|
| 244 |
+
"Fiber optic",
|
| 245 |
+
"No"
|
| 246 |
+
],
|
| 247 |
+
"Contract": [
|
| 248 |
+
"Month-to-month",
|
| 249 |
+
"One year",
|
| 250 |
+
"Two year"
|
| 251 |
+
],
|
| 252 |
+
"PaymentMethod": [
|
| 253 |
+
"Bank transfer",
|
| 254 |
+
"Credit card",
|
| 255 |
+
"Electronic check",
|
| 256 |
+
"Mailed check"
|
| 257 |
+
]
|
| 258 |
+
},
|
| 259 |
+
"defaults": {
|
| 260 |
+
"SeniorCitizen": 1.0,
|
| 261 |
+
"tenure": 37.0,
|
| 262 |
+
"MonthlyCharges": 70.26,
|
| 263 |
+
"TotalCharges": 2161.52,
|
| 264 |
+
"Partner": "Yes",
|
| 265 |
+
"Dependents": "No",
|
| 266 |
+
"PhoneService": "No",
|
| 267 |
+
"InternetService": "No",
|
| 268 |
+
"Contract": "Month-to-month",
|
| 269 |
+
"PaymentMethod": "Credit card"
|
| 270 |
+
},
|
| 271 |
+
"threshold": 0.1,
|
| 272 |
+
"training_rows": 3763,
|
| 273 |
+
"validation_rows": 941,
|
| 274 |
+
"test_rows": 1176,
|
| 275 |
+
"audit": {
|
| 276 |
+
"original_rows": 5880,
|
| 277 |
+
"duplicate_rows_removed": 0,
|
| 278 |
+
"rows_after_deduplication": 5880,
|
| 279 |
+
"positive_rate": 0.49286,
|
| 280 |
+
"feature_missing_counts": {
|
| 281 |
+
"SeniorCitizen": 0,
|
| 282 |
+
"tenure": 0,
|
| 283 |
+
"MonthlyCharges": 0,
|
| 284 |
+
"TotalCharges": 0,
|
| 285 |
+
"Partner": 0,
|
| 286 |
+
"Dependents": 0,
|
| 287 |
+
"PhoneService": 0,
|
| 288 |
+
"InternetService": 0,
|
| 289 |
+
"Contract": 0,
|
| 290 |
+
"PaymentMethod": 0
|
| 291 |
+
},
|
| 292 |
+
"sha256_first12": "5fa48a8d1aa1"
|
| 293 |
+
},
|
| 294 |
+
"validation_metrics": {
|
| 295 |
+
"roc_auc": 0.49782,
|
| 296 |
+
"pr_auc": 0.49893,
|
| 297 |
+
"balanced_accuracy": 0.5,
|
| 298 |
+
"precision": 0.49309,
|
| 299 |
+
"recall": 1.0,
|
| 300 |
+
"f1": 0.6605,
|
| 301 |
+
"f2": 0.82946,
|
| 302 |
+
"confusion_matrix": [
|
| 303 |
+
[
|
| 304 |
+
0,
|
| 305 |
+
477
|
| 306 |
+
],
|
| 307 |
+
[
|
| 308 |
+
0,
|
| 309 |
+
464
|
| 310 |
+
]
|
| 311 |
+
],
|
| 312 |
+
"classification_report": {
|
| 313 |
+
"0": {
|
| 314 |
+
"precision": 0.0,
|
| 315 |
+
"recall": 0.0,
|
| 316 |
+
"f1-score": 0.0,
|
| 317 |
+
"support": 477.0
|
| 318 |
+
},
|
| 319 |
+
"1": {
|
| 320 |
+
"precision": 0.49309245483528164,
|
| 321 |
+
"recall": 1.0,
|
| 322 |
+
"f1-score": 0.6604982206405694,
|
| 323 |
+
"support": 464.0
|
| 324 |
+
},
|
| 325 |
+
"accuracy": 0.49309245483528164,
|
| 326 |
+
"macro avg": {
|
| 327 |
+
"precision": 0.24654622741764082,
|
| 328 |
+
"recall": 0.5,
|
| 329 |
+
"f1-score": 0.3302491103202847,
|
| 330 |
+
"support": 941.0
|
| 331 |
+
},
|
| 332 |
+
"weighted avg": {
|
| 333 |
+
"precision": 0.24314016901548424,
|
| 334 |
+
"recall": 0.49309245483528164,
|
| 335 |
+
"f1-score": 0.3256866890299938,
|
| 336 |
+
"support": 941.0
|
| 337 |
+
}
|
| 338 |
+
}
|
| 339 |
+
},
|
| 340 |
+
"test_metrics": {
|
| 341 |
+
"roc_auc": 0.47952,
|
| 342 |
+
"pr_auc": 0.4709,
|
| 343 |
+
"balanced_accuracy": 0.5,
|
| 344 |
+
"precision": 0.4932,
|
| 345 |
+
"recall": 1.0,
|
| 346 |
+
"f1": 0.66059,
|
| 347 |
+
"f2": 0.82952,
|
| 348 |
+
"confusion_matrix": [
|
| 349 |
+
[
|
| 350 |
+
0,
|
| 351 |
+
596
|
| 352 |
+
],
|
| 353 |
+
[
|
| 354 |
+
0,
|
| 355 |
+
580
|
| 356 |
+
]
|
| 357 |
+
],
|
| 358 |
+
"classification_report": {
|
| 359 |
+
"0": {
|
| 360 |
+
"precision": 0.0,
|
| 361 |
+
"recall": 0.0,
|
| 362 |
+
"f1-score": 0.0,
|
| 363 |
+
"support": 596.0
|
| 364 |
+
},
|
| 365 |
+
"1": {
|
| 366 |
+
"precision": 0.4931972789115646,
|
| 367 |
+
"recall": 1.0,
|
| 368 |
+
"f1-score": 0.6605922551252847,
|
| 369 |
+
"support": 580.0
|
| 370 |
+
},
|
| 371 |
+
"accuracy": 0.4931972789115646,
|
| 372 |
+
"macro avg": {
|
| 373 |
+
"precision": 0.2465986394557823,
|
| 374 |
+
"recall": 0.5,
|
| 375 |
+
"f1-score": 0.33029612756264237,
|
| 376 |
+
"support": 1176.0
|
| 377 |
+
},
|
| 378 |
+
"weighted avg": {
|
| 379 |
+
"precision": 0.24324355592577165,
|
| 380 |
+
"recall": 0.4931972789115646,
|
| 381 |
+
"f1-score": 0.3258023026978445,
|
| 382 |
+
"support": 1176.0
|
| 383 |
+
}
|
| 384 |
+
}
|
| 385 |
+
},
|
| 386 |
+
"deployment_status": "research_only_low_signal",
|
| 387 |
+
"model_size_mb": 0.02,
|
| 388 |
+
"validation_model_comparison": [
|
| 389 |
+
{
|
| 390 |
+
"dataset": "telecom",
|
| 391 |
+
"model": "XGBoost",
|
| 392 |
+
"validation_pr_auc": 0.49893,
|
| 393 |
+
"validation_roc_auc": 0.49782,
|
| 394 |
+
"tuning": "6x3 CV"
|
| 395 |
+
},
|
| 396 |
+
{
|
| 397 |
+
"dataset": "telecom",
|
| 398 |
+
"model": "Logistic regression",
|
| 399 |
+
"validation_pr_auc": 0.48094,
|
| 400 |
+
"validation_roc_auc": 0.49748,
|
| 401 |
+
"tuning": "none"
|
| 402 |
+
},
|
| 403 |
+
{
|
| 404 |
+
"dataset": "telecom",
|
| 405 |
+
"model": "Random forest",
|
| 406 |
+
"validation_pr_auc": 0.47485,
|
| 407 |
+
"validation_roc_auc": 0.47279,
|
| 408 |
+
"tuning": "none"
|
| 409 |
+
}
|
| 410 |
+
]
|
| 411 |
+
}
|
| 412 |
+
},
|
| 413 |
+
"decision_rule": "maximize validation F2 (recall-weighted), after selecting by validation PR AUC",
|
| 414 |
+
"probability_warning": "Scores are uncalibrated model probabilities; not validated as true churn probabilities."
|
| 415 |
+
}
|
models/bank_churn.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5a09f2ae9686f373e1f35bc5e14a7eff4bb40c2e672f8ce7c8e33a8d22888446
|
| 3 |
+
size 38621
|
models/telecom_churn.joblib
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:618003aa3338238af7a9cefb178b41fd623dc53d51e9b86e949097b482dd9bfd
|
| 3 |
+
size 21001
|
notebooks/Customer_Churn_Training_Walkthrough.ipynb
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"id": "49b7c97a",
|
| 6 |
+
"metadata": {},
|
| 7 |
+
"source": [
|
| 8 |
+
"# Customer Churn Model — Reproducible Training Walkthrough\n",
|
| 9 |
+
"\n",
|
| 10 |
+
"This public portfolio notebook trains two independent churn classifiers, audits data quality, compares three algorithms, and evaluates a final held-out test set. Upload the source CSVs when prompted. No data or secrets are committed."
|
| 11 |
+
]
|
| 12 |
+
},
|
| 13 |
+
{
|
| 14 |
+
"cell_type": "code",
|
| 15 |
+
"execution_count": null,
|
| 16 |
+
"id": "fe218b6d",
|
| 17 |
+
"metadata": {},
|
| 18 |
+
"outputs": [],
|
| 19 |
+
"source": [
|
| 20 |
+
"%pip -q install \"numpy==2.3.5\" \"pandas==2.2.3\" \"scikit-learn==1.8.0\" \"xgboost==3.1.3\" \"joblib==1.5.3\" \"matplotlib>=3.8,<4\""
|
| 21 |
+
]
|
| 22 |
+
},
|
| 23 |
+
{
|
| 24 |
+
"cell_type": "markdown",
|
| 25 |
+
"id": "5514f474",
|
| 26 |
+
"metadata": {},
|
| 27 |
+
"source": [
|
| 28 |
+
"## Upload your bank and telecom datasets\n",
|
| 29 |
+
"Select the same `bank_customer_churn.csv` and `customer_churn_data.csv` used in the full deployment notebook."
|
| 30 |
+
]
|
| 31 |
+
},
|
| 32 |
+
{
|
| 33 |
+
"cell_type": "code",
|
| 34 |
+
"execution_count": null,
|
| 35 |
+
"id": "111b29e0",
|
| 36 |
+
"metadata": {},
|
| 37 |
+
"outputs": [],
|
| 38 |
+
"source": [
|
| 39 |
+
"from pathlib import Path\n",
|
| 40 |
+
"import shutil\n",
|
| 41 |
+
"from google.colab import files\n",
|
| 42 |
+
"DATA=Path(\"/content/churn_inputs\"); DATA.mkdir(exist_ok=True)\n",
|
| 43 |
+
"uploaded=files.upload()\n",
|
| 44 |
+
"for name, raw in uploaded.items():\n",
|
| 45 |
+
" if name.lower().endswith(\".csv\"):\n",
|
| 46 |
+
" (DATA/Path(name).name).write_bytes(raw)\n",
|
| 47 |
+
"for canonical in [\"bank_customer_churn.csv\", \"customer_churn_data.csv\"]:\n",
|
| 48 |
+
" if not (DATA/canonical).exists():\n",
|
| 49 |
+
" alternates=sorted(DATA.glob(Path(canonical).stem+\"(*)\"+Path(canonical).suffix))\n",
|
| 50 |
+
" if alternates: shutil.copy2(alternates[0],DATA/canonical)\n",
|
| 51 |
+
" assert (DATA/canonical).exists(), f\"Missing {canonical}\"\n"
|
| 52 |
+
]
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
"cell_type": "markdown",
|
| 56 |
+
"id": "fa8035d6",
|
| 57 |
+
"metadata": {},
|
| 58 |
+
"source": [
|
| 59 |
+
"## Data-safe reproducible training and held-out evaluation\n",
|
| 60 |
+
"The `train.py` source below is the identical source used in the Hugging Face release. Target labels and identifiers are excluded from features; model selection and threshold tuning use only validation data."
|
| 61 |
+
]
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
"cell_type": "code",
|
| 65 |
+
"execution_count": null,
|
| 66 |
+
"id": "fab89f88",
|
| 67 |
+
"metadata": {},
|
| 68 |
+
"outputs": [],
|
| 69 |
+
"source": [
|
| 70 |
+
"from pathlib import Path\n",
|
| 71 |
+
"TRAIN_SOURCE = '\"\"\"Reproducible bank + telecom churn training; no customer row data is published.\"\"\"\\nfrom __future__ import annotations\\nimport argparse\\nimport sys\\nimport hashlib\\nimport json\\nfrom datetime import datetime, timezone\\nfrom pathlib import Path\\n\\nimport joblib\\nimport matplotlib\\nmatplotlib.use(\\'Agg\\')\\nimport matplotlib.pyplot as plt\\nimport numpy as np\\nimport pandas as pd\\nimport sklearn\\nimport xgboost\\nfrom sklearn.compose import ColumnTransformer\\nfrom sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.impute import SimpleImputer\\nfrom sklearn.inspection import permutation_importance\\nfrom sklearn.linear_model import LogisticRegression\\nfrom sklearn.metrics import (average_precision_score, balanced_accuracy_score,\\n classification_report, confusion_matrix, ConfusionMatrixDisplay, f1_score,\\n fbeta_score, precision_score, recall_score, roc_auc_score, roc_curve)\\nfrom sklearn.model_selection import RandomizedSearchCV, StratifiedKFold, train_test_split\\nfrom sklearn.pipeline import Pipeline\\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\\nfrom xgboost import XGBClassifier\\n\\nSEED = 42\\nSPECS = {\\n \\'bank\\': {\\n \\'filename\\': \\'bank_customer_churn.csv\\', \\'target\\': \\'Exited\\',\\n \\'numeric\\': [\\'CreditScore\\',\\'Age\\',\\'Tenure\\',\\'Balance\\',\\'NumOfProducts\\',\\'HasCrCard\\',\\'IsActiveMember\\',\\'EstimatedSalary\\'],\\n \\'categorical\\': [\\'Geography\\'], \\'excluded\\': [\\'CustomerId\\',\\'Surname\\'],\\n \\'display\\': \\'Bank customer churn\\',\\n },\\n \\'telecom\\': {\\n \\'filename\\': \\'customer_churn_data.csv\\', \\'target\\': \\'Churn\\',\\n \\'numeric\\': [\\'SeniorCitizen\\',\\'tenure\\',\\'MonthlyCharges\\',\\'TotalCharges\\'],\\n \\'categorical\\': [\\'Partner\\',\\'Dependents\\',\\'PhoneService\\',\\'InternetService\\',\\'Contract\\',\\'PaymentMethod\\'],\\n \\'excluded\\': [\\'customerID\\'], \\'display\\': \\'Telecom customer churn\\',\\n },\\n}\\n\\ndef pyvalue(v):\\n if isinstance(v, np.generic):\\n return v.item()\\n return v\\n\\ndef load_dataset(path, spec):\\n if not path.is_file():\\n raise FileNotFoundError(f\\'Required dataset missing: {path}. Upload the original CSV in Colab.\\')\\n raw = pd.read_csv(path)\\n required = spec[\\'numeric\\'] + spec[\\'categorical\\'] + [spec[\\'target\\']]\\n missing = [c for c in required if c not in raw.columns]\\n if missing:\\n raise ValueError(f\\'{path.name}: missing columns {missing}. Found {list(raw.columns)}\\')\\n original_rows = len(raw)\\n frame = raw.drop_duplicates().copy()\\n # Never use identifiers, surnames, or the label as predictors.\\n X = frame[spec[\\'numeric\\'] + spec[\\'categorical\\']].copy()\\n for col in spec[\\'numeric\\']:\\n X[col] = pd.to_numeric(X[col], errors=\\'coerce\\')\\n for col in spec[\\'categorical\\']:\\n X[col] = X[col].astype(\\'string\\').str.strip().replace(\\'\\', pd.NA)\\n X[col] = X[col].astype(object).where(X[col].notna(), np.nan)\\n if spec[\\'target\\'] == \\'Churn\\':\\n y = frame[spec[\\'target\\']].astype(\\'string\\').str.strip().str.lower().map({\\'yes\\': 1, \\'no\\': 0})\\n else:\\n y = pd.to_numeric(frame[spec[\\'target\\']], errors=\\'coerce\\')\\n if y.isna().any() or not set(y.unique()).issubset({0, 1}):\\n raise ValueError(f\\'{path.name}: target must be fully labeled binary 0/1 or Yes/No\\')\\n y = y.astype(\\'int64\\')\\n if y.value_counts().min() < 20:\\n raise ValueError(f\\'{path.name}: insufficient positive/negative examples\\')\\n audit = {\\n \\'original_rows\\': int(original_rows),\\n \\'duplicate_rows_removed\\': int(original_rows-len(frame)),\\n \\'rows_after_deduplication\\': int(len(frame)),\\n \\'positive_rate\\': round(float(y.mean()), 5),\\n \\'feature_missing_counts\\': {col: int(X[col].isna().sum()) for col in X.columns},\\n \\'sha256_first12\\': hashlib.sha256(path.read_bytes()).hexdigest()[:12],\\n }\\n return X, y, audit\\n\\ndef preprocessing(spec):\\n numerical = Pipeline([(\\'impute\\', SimpleImputer(strategy=\\'median\\')),\\n (\\'scale\\', StandardScaler())])\\n categorical = Pipeline([(\\'impute\\', SimpleImputer(strategy=\\'most_frequent\\')),\\n (\\'onehot\\', OneHotEncoder(handle_unknown=\\'ignore\\', sparse_output=False))])\\n return ColumnTransformer([(\\'numeric\\', numerical, spec[\\'numeric\\']),\\n (\\'categorical\\', categorical, spec[\\'categorical\\'])],\\n remainder=\\'drop\\')\\n\\ndef score_proba(model, X):\\n return model.predict_proba(X)[:, 1]\\n\\ndef metrics(y, p, threshold):\\n pred = (p >= threshold).astype(int)\\n return {\\n \\'roc_auc\\': round(float(roc_auc_score(y, p)), 5),\\n \\'pr_auc\\': round(float(average_precision_score(y, p)), 5),\\n \\'balanced_accuracy\\': round(float(balanced_accuracy_score(y, pred)), 5),\\n \\'precision\\': round(float(precision_score(y, pred, zero_division=0)), 5),\\n \\'recall\\': round(float(recall_score(y, pred, zero_division=0)), 5),\\n \\'f1\\': round(float(f1_score(y, pred, zero_division=0)), 5),\\n \\'f2\\': round(float(fbeta_score(y, pred, beta=2, zero_division=0)), 5),\\n \\'confusion_matrix\\': confusion_matrix(y, pred, labels=[0, 1]).tolist(),\\n \\'classification_report\\': classification_report(y, pred, labels=[0, 1],\\n output_dict=True, zero_division=0),\\n }\\n\\ndef tune_threshold(y, p):\\n thresholds = np.round(np.arange(0.10, 0.901, 0.025), 3)\\n values = [fbeta_score(y, (p >= t).astype(int), beta=2, zero_division=0) for t in thresholds]\\n return float(thresholds[int(np.argmax(values))])\\n\\ndef plot_diagnostics(project, key, ytest, ptest, threshold, importance_df):\\n report_dir = project / \\'reports\\'\\n pred = (ptest >= threshold).astype(int)\\n fig, ax = plt.subplots(figsize=(5, 4))\\n ConfusionMatrixDisplay.from_predictions(ytest, pred, display_labels=[\\'Retained\\', \\'Churned\\'],\\n cmap=\\'Blues\\', colorbar=False, ax=ax)\\n ax.set_title(f\\'{key.title()} — untouched test data\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_confusion_matrix.png\\', dpi=140)\\n plt.close(fig)\\n fpr, tpr, _ = roc_curve(ytest, ptest)\\n fig, ax = plt.subplots(figsize=(5.5, 4))\\n ax.plot(fpr, tpr, label=f\\'ROC AUC = {roc_auc_score(ytest, ptest):.3f}\\')\\n ax.plot([0, 1], [0, 1], linestyle=\\'--\\', label=\\'Chance\\')\\n ax.set(xlabel=\\'False positive rate\\', ylabel=\\'True positive rate\\', title=f\\'{key.title()} — test ROC\\')\\n ax.legend(loc=\\'lower right\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_roc.png\\', dpi=140)\\n plt.close(fig)\\n top = importance_df.head(10).sort_values(\\'importance_mean\\')\\n fig, ax = plt.subplots(figsize=(7, 4.5))\\n ax.barh(top[\\'feature\\'], top[\\'importance_mean\\'])\\n ax.set(xlabel=\\'Change in validation PR AUC after shuffling\\', title=f\\'{key.title()} — permutation importance\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_feature_importance.png\\', dpi=140)\\n plt.close(fig)\\n\\ndef train_one(project, source_dir, key, tuning_iterations):\\n spec = SPECS[key]\\n path = source_dir / spec[\\'filename\\']\\n X, y, audit = load_dataset(path, spec)\\n X_dev, X_test, y_dev, y_test = train_test_split(X, y, test_size=0.20,\\n random_state=SEED, stratify=y)\\n X_train, X_val, y_train, y_val = train_test_split(X_dev, y_dev, test_size=0.20,\\n random_state=SEED, stratify=y_dev)\\n minority_weight = float((y_train == 0).sum() / (y_train == 1).sum())\\n models = {\\n \\'Logistic regression\\': LogisticRegression(max_iter=1500, class_weight=\\'balanced\\', random_state=SEED),\\n \\'Random forest\\': RandomForestClassifier(n_estimators=170, min_samples_leaf=3,\\n class_weight=\\'balanced_subsample\\', n_jobs=-1, random_state=SEED),\\n }\\n candidates = {}\\n leaderboard = []\\n for name, estimator in models.items():\\n pipeline = Pipeline([(\\'preprocessor\\', preprocessing(spec)), (\\'model\\', estimator)])\\n pipeline.fit(X_train, y_train)\\n candidates[name] = pipeline\\n row = metrics(y_val, score_proba(pipeline, X_val), 0.5)\\n leaderboard.append({\\'dataset\\': key, \\'model\\': name, \\'validation_pr_auc\\': row[\\'pr_auc\\'],\\n \\'validation_roc_auc\\': row[\\'roc_auc\\'], \\'tuning\\': \\'none\\'})\\n xgb_pipe = Pipeline([(\\'preprocessor\\', preprocessing(spec)),\\n (\\'model\\', XGBClassifier(n_estimators=200, max_depth=3, learning_rate=0.05,\\n subsample=0.9, colsample_bytree=0.9, scale_pos_weight=minority_weight,\\n eval_metric=\\'logloss\\', tree_method=\\'hist\\', random_state=SEED, n_jobs=1))])\\n if tuning_iterations:\\n params = {\\n \\'model__n_estimators\\': [100, 180, 260],\\n \\'model__max_depth\\': [2, 3, 4],\\n \\'model__learning_rate\\': [0.025, 0.05, 0.1],\\n \\'model__subsample\\': [0.8, 1.0],\\n \\'model__colsample_bytree\\': [0.8, 1.0],\\n \\'model__min_child_weight\\': [1, 3],\\n }\\n search = RandomizedSearchCV(xgb_pipe, params, n_iter=tuning_iterations,\\n scoring=\\'average_precision\\', cv=StratifiedKFold(3, shuffle=True, random_state=SEED),\\n random_state=SEED, n_jobs=2, refit=True, verbose=0, error_score=\\'raise\\')\\n search.fit(X_train, y_train)\\n xgb_pipe = search.best_estimator_\\n print(f\\'[{key}] XGBoost best 3-fold training CV PR AUC: {search.best_score_:.4f}\\')\\n else:\\n xgb_pipe.fit(X_train, y_train)\\n candidates[\\'XGBoost\\'] = xgb_pipe\\n val_xgb = metrics(y_val, score_proba(xgb_pipe, X_val), 0.5)\\n leaderboard.append({\\'dataset\\': key, \\'model\\': \\'XGBoost\\',\\n \\'validation_pr_auc\\': val_xgb[\\'pr_auc\\'], \\'validation_roc_auc\\': val_xgb[\\'roc_auc\\'],\\n \\'tuning\\': f\\'{tuning_iterations}x3 CV\\' if tuning_iterations else \\'off\\'})\\n leaderboard.sort(key=lambda r: (r[\\'validation_pr_auc\\'], r[\\'validation_roc_auc\\']), reverse=True)\\n winner = leaderboard[0][\\'model\\']\\n champion = candidates[winner]\\n threshold = tune_threshold(y_val, score_proba(champion, X_val))\\n # The test partition has never been used to choose model, hyperparameters, or threshold.\\n test_scores = score_proba(champion, X_test)\\n test = metrics(y_test, test_scores, threshold)\\n # Deliberate launch-quality gate: near-chance classifiers remain research-only.\\n # Hold-out data are used for final model assessment, not training/selection.\\n reliable = (test[\\'roc_auc\\'] >= 0.60 and test[\\'pr_auc\\'] >= float(y_test.mean()) + 0.03)\\n deployment_status = \\'demonstration\\' if reliable else \\'research_only_low_signal\\'\\n if not reliable:\\n print(f\\'[{key}] WARNING: near-chance model; API will not issue actionable churn labels.\\')\\n val = metrics(y_val, score_proba(champion, X_val), threshold)\\n defaults = {c: float(X_train[c].median()) for c in spec[\\'numeric\\']}\\n categories = {}\\n for col in spec[\\'categorical\\']:\\n vals = X_train[col].dropna().astype(str)\\n categories[col] = sorted(vals.unique().tolist())\\n defaults[col] = str(vals.mode().iloc[0])\\n importance = permutation_importance(champion, X_val, y_val, scoring=\\'average_precision\\',\\n n_repeats=3, random_state=SEED, n_jobs=1)\\n importance_df = pd.DataFrame({\\'feature\\': spec[\\'numeric\\'] + spec[\\'categorical\\'],\\n \\'importance_mean\\': importance.importances_mean, \\'importance_std\\': importance.importances_std})\\n importance_df = importance_df.sort_values(\\'importance_mean\\', ascending=False)\\n importance_df.to_csv(project / \\'reports\\' / f\\'{key}_feature_importance.csv\\', index=False)\\n plot_diagnostics(project, key, y_test, test_scores, threshold, importance_df)\\n bundle = {\\'pipeline\\': champion, \\'threshold\\': threshold,\\n \\'numeric\\': spec[\\'numeric\\'], \\'categorical\\': spec[\\'categorical\\'], \\'features\\': spec[\\'numeric\\'] + spec[\\'categorical\\']}\\n path_model = project / \\'models\\' / f\\'{key}_churn.joblib\\'\\n joblib.dump(bundle, path_model, compress=3)\\n reloaded = joblib.load(path_model)\\n assert np.allclose(reloaded[\\'pipeline\\'].predict_proba(X_test.iloc[:5])[:,1], test_scores[:5])\\n metadata = {\\n \\'label\\': spec[\\'display\\'], \\'selected_model\\': winner,\\n \\'numeric\\': spec[\\'numeric\\'], \\'categorical\\': spec[\\'categorical\\'],\\n \\'features\\': spec[\\'numeric\\'] + spec[\\'categorical\\'],\\n \\'excluded_columns\\': spec[\\'excluded\\'], \\'categorical_choices\\': categories,\\n \\'defaults\\': defaults, \\'threshold\\': threshold,\\n \\'training_rows\\': len(X_train), \\'validation_rows\\': len(X_val), \\'test_rows\\': len(X_test),\\n \\'audit\\': audit, \\'validation_metrics\\': val, \\'test_metrics\\': test,\\n \\'deployment_status\\': deployment_status,\\n \\'model_size_mb\\': round(path_model.stat().st_size / 1024**2, 3),\\n \\'validation_model_comparison\\': leaderboard,\\n }\\n print(f\\'[{key}] Winner: {winner}, threshold: {threshold:.3f}, \\'\\n f\\'test PR AUC: {test[\"pr_auc\"]:.4f}, ROC AUC: {test[\"roc_auc\"]:.4f}, recall: {test[\"recall\"]:.4f}\\')\\n return metadata, leaderboard\\n\\ndef main(data_dir=\\'.\\', project_dir=\\'.\\', tuning_iterations=6):\\n project = Path(project_dir).resolve()\\n data_dir = Path(data_dir).resolve()\\n (project / \\'models\\').mkdir(parents=True, exist_ok=True)\\n (project / \\'reports\\').mkdir(parents=True, exist_ok=True)\\n results, leaderboard = {}, []\\n for key in SPECS:\\n print(f\\'\\\\n=== TRAINING {key.upper()} ===\\')\\n results[key], comparison = train_one(project, data_dir, key, tuning_iterations)\\n leaderboard.extend(comparison)\\n summary = {\\'generated_utc\\': datetime.now(timezone.utc).isoformat(),\\n \\'random_seed\\': SEED, \\'sklearn_version\\': sklearn.__version__,\\n \\'xgboost_version\\': xgboost.__version__, \\'python_version\\': sys.version.split()[0], \\'datasets\\': results,\\n \\'decision_rule\\': \\'maximize validation F2 (recall-weighted), after selecting by validation PR AUC\\',\\n \\'probability_warning\\': \\'Scores are uncalibrated model probabilities; not validated as true churn probabilities.\\'}\\n (project / \\'model_metadata.json\\').write_text(json.dumps(summary, indent=2, default=pyvalue), encoding=\\'utf-8\\')\\n pd.DataFrame(leaderboard).to_csv(project / \\'reports\\' / \\'model_comparison.csv\\', index=False)\\n print(\\'\\\\nSaved trained pipelines, evaluation, charts, and audit metadata to\\', project)\\n return summary\\n\\nif __name__ == \\'__main__\\':\\n parser = argparse.ArgumentParser()\\n parser.add_argument(\\'--data-dir\\', default=\\'.\\')\\n parser.add_argument(\\'--project-dir\\', default=\\'.\\')\\n parser.add_argument(\\'--tuning-iterations\\', type=int, default=6)\\n args = parser.parse_args()\\n main(args.data_dir, args.project_dir, args.tuning_iterations)\\n'\n",
|
| 72 |
+
"Path(\"train.py\").write_text(TRAIN_SOURCE, encoding=\"utf-8\")\n",
|
| 73 |
+
"import sklearn, xgboost\n",
|
| 74 |
+
"assert (sklearn.__version__,xgboost.__version__)==(\"1.8.0\",\"3.1.3\"), \"Restart Colab runtime after install and rerun cells\"\n"
|
| 75 |
+
]
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"cell_type": "code",
|
| 79 |
+
"execution_count": null,
|
| 80 |
+
"id": "097cd908",
|
| 81 |
+
"metadata": {},
|
| 82 |
+
"outputs": [],
|
| 83 |
+
"source": [
|
| 84 |
+
"import importlib.util\n",
|
| 85 |
+
"spec=importlib.util.spec_from_file_location(\"churn_training\", \"train.py\")\n",
|
| 86 |
+
"module=importlib.util.module_from_spec(spec)\n",
|
| 87 |
+
"spec.loader.exec_module(module)\n",
|
| 88 |
+
"results=module.main(data_dir=DATA,project_dir=Path(\"/content/churn-intelligence-lab\"),tuning_iterations=6)\n"
|
| 89 |
+
]
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"cell_type": "markdown",
|
| 93 |
+
"id": "185df73a",
|
| 94 |
+
"metadata": {},
|
| 95 |
+
"source": [
|
| 96 |
+
"## Compare model performance and diagnostics\n",
|
| 97 |
+
"The quality gate marks a near-chance classifier as research-only. The test partition is independent of validation-based selection."
|
| 98 |
+
]
|
| 99 |
+
},
|
| 100 |
+
{
|
| 101 |
+
"cell_type": "code",
|
| 102 |
+
"execution_count": null,
|
| 103 |
+
"id": "ffcd1d62",
|
| 104 |
+
"metadata": {},
|
| 105 |
+
"outputs": [],
|
| 106 |
+
"source": [
|
| 107 |
+
"from IPython.display import display, Image\n",
|
| 108 |
+
"import pandas as pd\n",
|
| 109 |
+
"P=Path(\"/content/churn-intelligence-lab\")\n",
|
| 110 |
+
"display(pd.read_csv(P/\"reports\"/\"model_comparison.csv\"))\n",
|
| 111 |
+
"for segment in (\"bank\",\"telecom\"):\n",
|
| 112 |
+
" print(segment.upper(),results[\"datasets\"][segment][\"test_metrics\"])\n",
|
| 113 |
+
" display(Image(filename=str(P/\"reports\"/f\"{segment}_roc.png\"),width=550))\n",
|
| 114 |
+
" display(Image(filename=str(P/\"reports\"/f\"{segment}_feature_importance.png\"),width=550))"
|
| 115 |
+
]
|
| 116 |
+
}
|
| 117 |
+
],
|
| 118 |
+
"metadata": {
|
| 119 |
+
"colab": {
|
| 120 |
+
"provenance": []
|
| 121 |
+
},
|
| 122 |
+
"kernelspec": {
|
| 123 |
+
"display_name": "Python 3",
|
| 124 |
+
"name": "python3"
|
| 125 |
+
}
|
| 126 |
+
},
|
| 127 |
+
"nbformat": 4,
|
| 128 |
+
"nbformat_minor": 5
|
| 129 |
+
}
|
reports/bank_confusion_matrix.png
ADDED
|
reports/bank_feature_importance.csv
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
feature,importance_mean,importance_std
|
| 2 |
+
Age,0.26445187371299056,0.020430558670981074
|
| 3 |
+
NumOfProducts,0.24834748937180307,0.008799324747770149
|
| 4 |
+
IsActiveMember,0.09675602920420179,0.005445308581024779
|
| 5 |
+
Balance,0.0753602399656933,0.0046094855266077565
|
| 6 |
+
Geography,0.06101594766852877,0.0020921889588975118
|
| 7 |
+
CreditScore,0.004736756017056698,0.0006832834776869457
|
| 8 |
+
EstimatedSalary,0.0031848726894710966,0.001764580082232362
|
| 9 |
+
HasCrCard,-0.0002528830199832373,0.0005843858996471953
|
| 10 |
+
Tenure,-0.00034052985012456016,0.0020594317889659907
|
reports/bank_feature_importance.png
ADDED
|
reports/bank_roc.png
ADDED
|
reports/model_comparison.csv
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,validation_pr_auc,validation_roc_auc,tuning
|
| 2 |
+
bank,XGBoost,0.68442,0.85564,6x3 CV
|
| 3 |
+
bank,Random forest,0.68408,0.86424,none
|
| 4 |
+
bank,Logistic regression,0.45514,0.75422,none
|
| 5 |
+
telecom,XGBoost,0.49893,0.49782,6x3 CV
|
| 6 |
+
telecom,Logistic regression,0.48094,0.49748,none
|
| 7 |
+
telecom,Random forest,0.47485,0.47279,none
|
reports/telecom_confusion_matrix.png
ADDED
|
reports/telecom_feature_importance.csv
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
feature,importance_mean,importance_std
|
| 2 |
+
MonthlyCharges,0.01157943277453157,0.004786441332147423
|
| 3 |
+
PaymentMethod,0.0021447557360410228,0.0020976124157429167
|
| 4 |
+
Dependents,0.0005530449833317889,0.000222747091901983
|
| 5 |
+
Contract,0.0005015827719847173,0.0013145344028538154
|
| 6 |
+
SeniorCitizen,0.0,0.0
|
| 7 |
+
Partner,-0.00035698457050259425,0.0003064753166891028
|
| 8 |
+
PhoneService,-0.0005407161681062663,0.00033801364566065804
|
| 9 |
+
tenure,-0.001093924683967314,0.009684521252210336
|
| 10 |
+
TotalCharges,-0.006343578030426789,0.007965392546590323
|
| 11 |
+
InternetService,-0.00818567707629129,0.0003004453118950846
|
reports/telecom_feature_importance.png
ADDED
|
reports/telecom_roc.png
ADDED
|
requirements.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy==2.3.5
|
| 2 |
+
pandas==2.2.3
|
| 3 |
+
scikit-learn==1.8.0
|
| 4 |
+
xgboost==3.1.3
|
| 5 |
+
joblib==1.5.3
|
| 6 |
+
fastapi==0.128.2
|
| 7 |
+
gradio==6.5.1
|
| 8 |
+
uvicorn==0.48.0
|
| 9 |
+
pydantic==2.13.5
|
tests/test_api.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Run after training: python -m pytest -q tests/"""
|
| 2 |
+
import json
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
from fastapi.testclient import TestClient
|
| 5 |
+
from app import app, META, predict_records
|
| 6 |
+
|
| 7 |
+
client = TestClient(app)
|
| 8 |
+
|
| 9 |
+
def test_health():
|
| 10 |
+
r = client.get('/health')
|
| 11 |
+
assert r.status_code == 200
|
| 12 |
+
assert set(r.json()['trained_models']) == {'bank', 'telecom'}
|
| 13 |
+
|
| 14 |
+
def test_ui_redirect():
|
| 15 |
+
r = client.get('/', follow_redirects=False)
|
| 16 |
+
assert r.status_code in (302, 307, 308)
|
| 17 |
+
assert r.headers['location'] == '/ui/'
|
| 18 |
+
|
| 19 |
+
def test_predictions():
|
| 20 |
+
for segment, info in META.items():
|
| 21 |
+
sample = info['defaults']
|
| 22 |
+
r = client.post(f'/api/predict/{segment}', json={'data': sample})
|
| 23 |
+
assert r.status_code == 200, r.text
|
| 24 |
+
result = r.json()
|
| 25 |
+
assert 0 <= result['churn_score'] <= 1
|
| 26 |
+
assert result['prediction'] in ('Likely to churn', 'Likely to stay',
|
| 27 |
+
'Exploratory only — unreliable classifier')
|
| 28 |
+
if segment == 'telecom' and info['deployment_status'] != 'demonstration':
|
| 29 |
+
assert result['prediction'].startswith('Exploratory only')
|
| 30 |
+
r_batch = client.post(f'/api/predict-batch/{segment}', json={'records': [sample, sample]})
|
| 31 |
+
assert r_batch.status_code == 200, r_batch.text
|
| 32 |
+
assert len(r_batch.json()['results']) == 2
|
| 33 |
+
assert r_batch.json()['results'][0]['churn_score'] == result['churn_score']
|
| 34 |
+
|
| 35 |
+
def test_invalid_fields_rejected():
|
| 36 |
+
r = client.post('/api/predict/bank', json={'data': {'Age': 40}})
|
| 37 |
+
assert r.status_code == 422
|
| 38 |
+
r2 = client.post('/api/predict/not-a-model', json={'data': META['bank']['defaults']})
|
| 39 |
+
assert r2.status_code == 422
|
| 40 |
+
|
| 41 |
+
def test_no_identifiers_in_model_schema():
|
| 42 |
+
for name, info in META.items():
|
| 43 |
+
assert not set(info['features']) & set(info['excluded_columns'])
|
| 44 |
+
assert len(info['features']) >= 9
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
def test_swagger_documentation():
|
| 48 |
+
response = client.get('/openapi.json')
|
| 49 |
+
assert response.status_code == 200
|
| 50 |
+
assert '/api/predict/bank' not in response.json()['paths'] # routing uses path template
|
| 51 |
+
assert '/api/predict/{segment}' in response.json()['paths']
|
| 52 |
+
|
| 53 |
+
def test_unexpected_category_or_nonfinite_rejected():
|
| 54 |
+
bank = dict(META['bank']['defaults'])
|
| 55 |
+
bank['Geography'] = 'UNLISTED'
|
| 56 |
+
assert client.post('/api/predict/bank', json={'data':bank}).status_code == 422
|
| 57 |
+
bank = dict(META['bank']['defaults'])
|
| 58 |
+
bank['Age'] = 'not-a-number'
|
| 59 |
+
assert client.post('/api/predict/bank', json={'data':bank}).status_code == 422
|
| 60 |
+
|
| 61 |
+
def test_batch_above_limit_rejected():
|
| 62 |
+
payload = {'records':[dict(META['bank']['defaults']) for _ in range(101)]}
|
| 63 |
+
assert client.post('/api/predict-batch/bank', json=payload).status_code == 422
|
train.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Reproducible bank + telecom churn training; no customer row data is published."""
|
| 2 |
+
from __future__ import annotations
|
| 3 |
+
import argparse
|
| 4 |
+
import sys
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
from datetime import datetime, timezone
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
import joblib
|
| 11 |
+
import matplotlib
|
| 12 |
+
matplotlib.use('Agg')
|
| 13 |
+
import matplotlib.pyplot as plt
|
| 14 |
+
import numpy as np
|
| 15 |
+
import pandas as pd
|
| 16 |
+
import sklearn
|
| 17 |
+
import xgboost
|
| 18 |
+
from sklearn.compose import ColumnTransformer
|
| 19 |
+
from sklearn.ensemble import RandomForestClassifier
|
| 20 |
+
from sklearn.impute import SimpleImputer
|
| 21 |
+
from sklearn.inspection import permutation_importance
|
| 22 |
+
from sklearn.linear_model import LogisticRegression
|
| 23 |
+
from sklearn.metrics import (average_precision_score, balanced_accuracy_score,
|
| 24 |
+
classification_report, confusion_matrix, ConfusionMatrixDisplay, f1_score,
|
| 25 |
+
fbeta_score, precision_score, recall_score, roc_auc_score, roc_curve)
|
| 26 |
+
from sklearn.model_selection import RandomizedSearchCV, StratifiedKFold, train_test_split
|
| 27 |
+
from sklearn.pipeline import Pipeline
|
| 28 |
+
from sklearn.preprocessing import OneHotEncoder, StandardScaler
|
| 29 |
+
from xgboost import XGBClassifier
|
| 30 |
+
|
| 31 |
+
SEED = 42
|
| 32 |
+
SPECS = {
|
| 33 |
+
'bank': {
|
| 34 |
+
'filename': 'bank_customer_churn.csv', 'target': 'Exited',
|
| 35 |
+
'numeric': ['CreditScore','Age','Tenure','Balance','NumOfProducts','HasCrCard','IsActiveMember','EstimatedSalary'],
|
| 36 |
+
'categorical': ['Geography'], 'excluded': ['CustomerId','Surname'],
|
| 37 |
+
'display': 'Bank customer churn',
|
| 38 |
+
},
|
| 39 |
+
'telecom': {
|
| 40 |
+
'filename': 'customer_churn_data.csv', 'target': 'Churn',
|
| 41 |
+
'numeric': ['SeniorCitizen','tenure','MonthlyCharges','TotalCharges'],
|
| 42 |
+
'categorical': ['Partner','Dependents','PhoneService','InternetService','Contract','PaymentMethod'],
|
| 43 |
+
'excluded': ['customerID'], 'display': 'Telecom customer churn',
|
| 44 |
+
},
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
def pyvalue(v):
|
| 48 |
+
if isinstance(v, np.generic):
|
| 49 |
+
return v.item()
|
| 50 |
+
return v
|
| 51 |
+
|
| 52 |
+
def load_dataset(path, spec):
|
| 53 |
+
if not path.is_file():
|
| 54 |
+
raise FileNotFoundError(f'Required dataset missing: {path}. Upload the original CSV in Colab.')
|
| 55 |
+
raw = pd.read_csv(path)
|
| 56 |
+
required = spec['numeric'] + spec['categorical'] + [spec['target']]
|
| 57 |
+
missing = [c for c in required if c not in raw.columns]
|
| 58 |
+
if missing:
|
| 59 |
+
raise ValueError(f'{path.name}: missing columns {missing}. Found {list(raw.columns)}')
|
| 60 |
+
original_rows = len(raw)
|
| 61 |
+
frame = raw.drop_duplicates().copy()
|
| 62 |
+
# Never use identifiers, surnames, or the label as predictors.
|
| 63 |
+
X = frame[spec['numeric'] + spec['categorical']].copy()
|
| 64 |
+
for col in spec['numeric']:
|
| 65 |
+
X[col] = pd.to_numeric(X[col], errors='coerce')
|
| 66 |
+
for col in spec['categorical']:
|
| 67 |
+
X[col] = X[col].astype('string').str.strip().replace('', pd.NA)
|
| 68 |
+
X[col] = X[col].astype(object).where(X[col].notna(), np.nan)
|
| 69 |
+
if spec['target'] == 'Churn':
|
| 70 |
+
y = frame[spec['target']].astype('string').str.strip().str.lower().map({'yes': 1, 'no': 0})
|
| 71 |
+
else:
|
| 72 |
+
y = pd.to_numeric(frame[spec['target']], errors='coerce')
|
| 73 |
+
if y.isna().any() or not set(y.unique()).issubset({0, 1}):
|
| 74 |
+
raise ValueError(f'{path.name}: target must be fully labeled binary 0/1 or Yes/No')
|
| 75 |
+
y = y.astype('int64')
|
| 76 |
+
if y.value_counts().min() < 20:
|
| 77 |
+
raise ValueError(f'{path.name}: insufficient positive/negative examples')
|
| 78 |
+
audit = {
|
| 79 |
+
'original_rows': int(original_rows),
|
| 80 |
+
'duplicate_rows_removed': int(original_rows-len(frame)),
|
| 81 |
+
'rows_after_deduplication': int(len(frame)),
|
| 82 |
+
'positive_rate': round(float(y.mean()), 5),
|
| 83 |
+
'feature_missing_counts': {col: int(X[col].isna().sum()) for col in X.columns},
|
| 84 |
+
'sha256_first12': hashlib.sha256(path.read_bytes()).hexdigest()[:12],
|
| 85 |
+
}
|
| 86 |
+
return X, y, audit
|
| 87 |
+
|
| 88 |
+
def preprocessing(spec):
|
| 89 |
+
numerical = Pipeline([('impute', SimpleImputer(strategy='median')),
|
| 90 |
+
('scale', StandardScaler())])
|
| 91 |
+
categorical = Pipeline([('impute', SimpleImputer(strategy='most_frequent')),
|
| 92 |
+
('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))])
|
| 93 |
+
return ColumnTransformer([('numeric', numerical, spec['numeric']),
|
| 94 |
+
('categorical', categorical, spec['categorical'])],
|
| 95 |
+
remainder='drop')
|
| 96 |
+
|
| 97 |
+
def score_proba(model, X):
|
| 98 |
+
return model.predict_proba(X)[:, 1]
|
| 99 |
+
|
| 100 |
+
def metrics(y, p, threshold):
|
| 101 |
+
pred = (p >= threshold).astype(int)
|
| 102 |
+
return {
|
| 103 |
+
'roc_auc': round(float(roc_auc_score(y, p)), 5),
|
| 104 |
+
'pr_auc': round(float(average_precision_score(y, p)), 5),
|
| 105 |
+
'balanced_accuracy': round(float(balanced_accuracy_score(y, pred)), 5),
|
| 106 |
+
'precision': round(float(precision_score(y, pred, zero_division=0)), 5),
|
| 107 |
+
'recall': round(float(recall_score(y, pred, zero_division=0)), 5),
|
| 108 |
+
'f1': round(float(f1_score(y, pred, zero_division=0)), 5),
|
| 109 |
+
'f2': round(float(fbeta_score(y, pred, beta=2, zero_division=0)), 5),
|
| 110 |
+
'confusion_matrix': confusion_matrix(y, pred, labels=[0, 1]).tolist(),
|
| 111 |
+
'classification_report': classification_report(y, pred, labels=[0, 1],
|
| 112 |
+
output_dict=True, zero_division=0),
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
def tune_threshold(y, p):
|
| 116 |
+
thresholds = np.round(np.arange(0.10, 0.901, 0.025), 3)
|
| 117 |
+
values = [fbeta_score(y, (p >= t).astype(int), beta=2, zero_division=0) for t in thresholds]
|
| 118 |
+
return float(thresholds[int(np.argmax(values))])
|
| 119 |
+
|
| 120 |
+
def plot_diagnostics(project, key, ytest, ptest, threshold, importance_df):
|
| 121 |
+
report_dir = project / 'reports'
|
| 122 |
+
pred = (ptest >= threshold).astype(int)
|
| 123 |
+
fig, ax = plt.subplots(figsize=(5, 4))
|
| 124 |
+
ConfusionMatrixDisplay.from_predictions(ytest, pred, display_labels=['Retained', 'Churned'],
|
| 125 |
+
cmap='Blues', colorbar=False, ax=ax)
|
| 126 |
+
ax.set_title(f'{key.title()} — untouched test data')
|
| 127 |
+
fig.tight_layout()
|
| 128 |
+
fig.savefig(report_dir / f'{key}_confusion_matrix.png', dpi=140)
|
| 129 |
+
plt.close(fig)
|
| 130 |
+
fpr, tpr, _ = roc_curve(ytest, ptest)
|
| 131 |
+
fig, ax = plt.subplots(figsize=(5.5, 4))
|
| 132 |
+
ax.plot(fpr, tpr, label=f'ROC AUC = {roc_auc_score(ytest, ptest):.3f}')
|
| 133 |
+
ax.plot([0, 1], [0, 1], linestyle='--', label='Chance')
|
| 134 |
+
ax.set(xlabel='False positive rate', ylabel='True positive rate', title=f'{key.title()} — test ROC')
|
| 135 |
+
ax.legend(loc='lower right')
|
| 136 |
+
fig.tight_layout()
|
| 137 |
+
fig.savefig(report_dir / f'{key}_roc.png', dpi=140)
|
| 138 |
+
plt.close(fig)
|
| 139 |
+
top = importance_df.head(10).sort_values('importance_mean')
|
| 140 |
+
fig, ax = plt.subplots(figsize=(7, 4.5))
|
| 141 |
+
ax.barh(top['feature'], top['importance_mean'])
|
| 142 |
+
ax.set(xlabel='Change in validation PR AUC after shuffling', title=f'{key.title()} — permutation importance')
|
| 143 |
+
fig.tight_layout()
|
| 144 |
+
fig.savefig(report_dir / f'{key}_feature_importance.png', dpi=140)
|
| 145 |
+
plt.close(fig)
|
| 146 |
+
|
| 147 |
+
def train_one(project, source_dir, key, tuning_iterations):
|
| 148 |
+
spec = SPECS[key]
|
| 149 |
+
path = source_dir / spec['filename']
|
| 150 |
+
X, y, audit = load_dataset(path, spec)
|
| 151 |
+
X_dev, X_test, y_dev, y_test = train_test_split(X, y, test_size=0.20,
|
| 152 |
+
random_state=SEED, stratify=y)
|
| 153 |
+
X_train, X_val, y_train, y_val = train_test_split(X_dev, y_dev, test_size=0.20,
|
| 154 |
+
random_state=SEED, stratify=y_dev)
|
| 155 |
+
minority_weight = float((y_train == 0).sum() / (y_train == 1).sum())
|
| 156 |
+
models = {
|
| 157 |
+
'Logistic regression': LogisticRegression(max_iter=1500, class_weight='balanced', random_state=SEED),
|
| 158 |
+
'Random forest': RandomForestClassifier(n_estimators=170, min_samples_leaf=3,
|
| 159 |
+
class_weight='balanced_subsample', n_jobs=-1, random_state=SEED),
|
| 160 |
+
}
|
| 161 |
+
candidates = {}
|
| 162 |
+
leaderboard = []
|
| 163 |
+
for name, estimator in models.items():
|
| 164 |
+
pipeline = Pipeline([('preprocessor', preprocessing(spec)), ('model', estimator)])
|
| 165 |
+
pipeline.fit(X_train, y_train)
|
| 166 |
+
candidates[name] = pipeline
|
| 167 |
+
row = metrics(y_val, score_proba(pipeline, X_val), 0.5)
|
| 168 |
+
leaderboard.append({'dataset': key, 'model': name, 'validation_pr_auc': row['pr_auc'],
|
| 169 |
+
'validation_roc_auc': row['roc_auc'], 'tuning': 'none'})
|
| 170 |
+
xgb_pipe = Pipeline([('preprocessor', preprocessing(spec)),
|
| 171 |
+
('model', XGBClassifier(n_estimators=200, max_depth=3, learning_rate=0.05,
|
| 172 |
+
subsample=0.9, colsample_bytree=0.9, scale_pos_weight=minority_weight,
|
| 173 |
+
eval_metric='logloss', tree_method='hist', random_state=SEED, n_jobs=1))])
|
| 174 |
+
if tuning_iterations:
|
| 175 |
+
params = {
|
| 176 |
+
'model__n_estimators': [100, 180, 260],
|
| 177 |
+
'model__max_depth': [2, 3, 4],
|
| 178 |
+
'model__learning_rate': [0.025, 0.05, 0.1],
|
| 179 |
+
'model__subsample': [0.8, 1.0],
|
| 180 |
+
'model__colsample_bytree': [0.8, 1.0],
|
| 181 |
+
'model__min_child_weight': [1, 3],
|
| 182 |
+
}
|
| 183 |
+
search = RandomizedSearchCV(xgb_pipe, params, n_iter=tuning_iterations,
|
| 184 |
+
scoring='average_precision', cv=StratifiedKFold(3, shuffle=True, random_state=SEED),
|
| 185 |
+
random_state=SEED, n_jobs=2, refit=True, verbose=0, error_score='raise')
|
| 186 |
+
search.fit(X_train, y_train)
|
| 187 |
+
xgb_pipe = search.best_estimator_
|
| 188 |
+
print(f'[{key}] XGBoost best 3-fold training CV PR AUC: {search.best_score_:.4f}')
|
| 189 |
+
else:
|
| 190 |
+
xgb_pipe.fit(X_train, y_train)
|
| 191 |
+
candidates['XGBoost'] = xgb_pipe
|
| 192 |
+
val_xgb = metrics(y_val, score_proba(xgb_pipe, X_val), 0.5)
|
| 193 |
+
leaderboard.append({'dataset': key, 'model': 'XGBoost',
|
| 194 |
+
'validation_pr_auc': val_xgb['pr_auc'], 'validation_roc_auc': val_xgb['roc_auc'],
|
| 195 |
+
'tuning': f'{tuning_iterations}x3 CV' if tuning_iterations else 'off'})
|
| 196 |
+
leaderboard.sort(key=lambda r: (r['validation_pr_auc'], r['validation_roc_auc']), reverse=True)
|
| 197 |
+
winner = leaderboard[0]['model']
|
| 198 |
+
champion = candidates[winner]
|
| 199 |
+
threshold = tune_threshold(y_val, score_proba(champion, X_val))
|
| 200 |
+
# The test partition has never been used to choose model, hyperparameters, or threshold.
|
| 201 |
+
test_scores = score_proba(champion, X_test)
|
| 202 |
+
test = metrics(y_test, test_scores, threshold)
|
| 203 |
+
# Deliberate launch-quality gate: near-chance classifiers remain research-only.
|
| 204 |
+
# Hold-out data are used for final model assessment, not training/selection.
|
| 205 |
+
reliable = (test['roc_auc'] >= 0.60 and test['pr_auc'] >= float(y_test.mean()) + 0.03)
|
| 206 |
+
deployment_status = 'demonstration' if reliable else 'research_only_low_signal'
|
| 207 |
+
if not reliable:
|
| 208 |
+
print(f'[{key}] WARNING: near-chance model; API will not issue actionable churn labels.')
|
| 209 |
+
val = metrics(y_val, score_proba(champion, X_val), threshold)
|
| 210 |
+
defaults = {c: float(X_train[c].median()) for c in spec['numeric']}
|
| 211 |
+
categories = {}
|
| 212 |
+
for col in spec['categorical']:
|
| 213 |
+
vals = X_train[col].dropna().astype(str)
|
| 214 |
+
categories[col] = sorted(vals.unique().tolist())
|
| 215 |
+
defaults[col] = str(vals.mode().iloc[0])
|
| 216 |
+
importance = permutation_importance(champion, X_val, y_val, scoring='average_precision',
|
| 217 |
+
n_repeats=3, random_state=SEED, n_jobs=1)
|
| 218 |
+
importance_df = pd.DataFrame({'feature': spec['numeric'] + spec['categorical'],
|
| 219 |
+
'importance_mean': importance.importances_mean, 'importance_std': importance.importances_std})
|
| 220 |
+
importance_df = importance_df.sort_values('importance_mean', ascending=False)
|
| 221 |
+
importance_df.to_csv(project / 'reports' / f'{key}_feature_importance.csv', index=False)
|
| 222 |
+
plot_diagnostics(project, key, y_test, test_scores, threshold, importance_df)
|
| 223 |
+
bundle = {'pipeline': champion, 'threshold': threshold,
|
| 224 |
+
'numeric': spec['numeric'], 'categorical': spec['categorical'], 'features': spec['numeric'] + spec['categorical']}
|
| 225 |
+
path_model = project / 'models' / f'{key}_churn.joblib'
|
| 226 |
+
joblib.dump(bundle, path_model, compress=3)
|
| 227 |
+
reloaded = joblib.load(path_model)
|
| 228 |
+
assert np.allclose(reloaded['pipeline'].predict_proba(X_test.iloc[:5])[:,1], test_scores[:5])
|
| 229 |
+
metadata = {
|
| 230 |
+
'label': spec['display'], 'selected_model': winner,
|
| 231 |
+
'numeric': spec['numeric'], 'categorical': spec['categorical'],
|
| 232 |
+
'features': spec['numeric'] + spec['categorical'],
|
| 233 |
+
'excluded_columns': spec['excluded'], 'categorical_choices': categories,
|
| 234 |
+
'defaults': defaults, 'threshold': threshold,
|
| 235 |
+
'training_rows': len(X_train), 'validation_rows': len(X_val), 'test_rows': len(X_test),
|
| 236 |
+
'audit': audit, 'validation_metrics': val, 'test_metrics': test,
|
| 237 |
+
'deployment_status': deployment_status,
|
| 238 |
+
'model_size_mb': round(path_model.stat().st_size / 1024**2, 3),
|
| 239 |
+
'validation_model_comparison': leaderboard,
|
| 240 |
+
}
|
| 241 |
+
print(f'[{key}] Winner: {winner}, threshold: {threshold:.3f}, '
|
| 242 |
+
f'test PR AUC: {test["pr_auc"]:.4f}, ROC AUC: {test["roc_auc"]:.4f}, recall: {test["recall"]:.4f}')
|
| 243 |
+
return metadata, leaderboard
|
| 244 |
+
|
| 245 |
+
def main(data_dir='.', project_dir='.', tuning_iterations=6):
|
| 246 |
+
project = Path(project_dir).resolve()
|
| 247 |
+
data_dir = Path(data_dir).resolve()
|
| 248 |
+
(project / 'models').mkdir(parents=True, exist_ok=True)
|
| 249 |
+
(project / 'reports').mkdir(parents=True, exist_ok=True)
|
| 250 |
+
results, leaderboard = {}, []
|
| 251 |
+
for key in SPECS:
|
| 252 |
+
print(f'\n=== TRAINING {key.upper()} ===')
|
| 253 |
+
results[key], comparison = train_one(project, data_dir, key, tuning_iterations)
|
| 254 |
+
leaderboard.extend(comparison)
|
| 255 |
+
summary = {'generated_utc': datetime.now(timezone.utc).isoformat(),
|
| 256 |
+
'random_seed': SEED, 'sklearn_version': sklearn.__version__,
|
| 257 |
+
'xgboost_version': xgboost.__version__, 'python_version': sys.version.split()[0], 'datasets': results,
|
| 258 |
+
'decision_rule': 'maximize validation F2 (recall-weighted), after selecting by validation PR AUC',
|
| 259 |
+
'probability_warning': 'Scores are uncalibrated model probabilities; not validated as true churn probabilities.'}
|
| 260 |
+
(project / 'model_metadata.json').write_text(json.dumps(summary, indent=2, default=pyvalue), encoding='utf-8')
|
| 261 |
+
pd.DataFrame(leaderboard).to_csv(project / 'reports' / 'model_comparison.csv', index=False)
|
| 262 |
+
print('\nSaved trained pipelines, evaluation, charts, and audit metadata to', project)
|
| 263 |
+
return summary
|
| 264 |
+
|
| 265 |
+
if __name__ == '__main__':
|
| 266 |
+
parser = argparse.ArgumentParser()
|
| 267 |
+
parser.add_argument('--data-dir', default='.')
|
| 268 |
+
parser.add_argument('--project-dir', default='.')
|
| 269 |
+
parser.add_argument('--tuning-iterations', type=int, default=6)
|
| 270 |
+
args = parser.parse_args()
|
| 271 |
+
main(args.data_dir, args.project_dir, args.tuning_iterations)
|