File size: 20,255 Bytes
422809f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | {
"cells": [
{
"cell_type": "markdown",
"id": "49b7c97a",
"metadata": {},
"source": [
"# Customer Churn Model — Reproducible Training Walkthrough\n",
"\n",
"This public portfolio notebook trains two independent churn classifiers, audits data quality, compares three algorithms, and evaluates a final held-out test set. Upload the source CSVs when prompted. No data or secrets are committed."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fe218b6d",
"metadata": {},
"outputs": [],
"source": [
"%pip -q install \"numpy==2.3.5\" \"pandas==2.2.3\" \"scikit-learn==1.8.0\" \"xgboost==3.1.3\" \"joblib==1.5.3\" \"matplotlib>=3.8,<4\""
]
},
{
"cell_type": "markdown",
"id": "5514f474",
"metadata": {},
"source": [
"## Upload your bank and telecom datasets\n",
"Select the same `bank_customer_churn.csv` and `customer_churn_data.csv` used in the full deployment notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "111b29e0",
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"import shutil\n",
"from google.colab import files\n",
"DATA=Path(\"/content/churn_inputs\"); DATA.mkdir(exist_ok=True)\n",
"uploaded=files.upload()\n",
"for name, raw in uploaded.items():\n",
" if name.lower().endswith(\".csv\"):\n",
" (DATA/Path(name).name).write_bytes(raw)\n",
"for canonical in [\"bank_customer_churn.csv\", \"customer_churn_data.csv\"]:\n",
" if not (DATA/canonical).exists():\n",
" alternates=sorted(DATA.glob(Path(canonical).stem+\"(*)\"+Path(canonical).suffix))\n",
" if alternates: shutil.copy2(alternates[0],DATA/canonical)\n",
" assert (DATA/canonical).exists(), f\"Missing {canonical}\"\n"
]
},
{
"cell_type": "markdown",
"id": "fa8035d6",
"metadata": {},
"source": [
"## Data-safe reproducible training and held-out evaluation\n",
"The `train.py` source below is the identical source used in the Hugging Face release. Target labels and identifiers are excluded from features; model selection and threshold tuning use only validation data."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fab89f88",
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"TRAIN_SOURCE = '\"\"\"Reproducible bank + telecom churn training; no customer row data is published.\"\"\"\\nfrom __future__ import annotations\\nimport argparse\\nimport sys\\nimport hashlib\\nimport json\\nfrom datetime import datetime, timezone\\nfrom pathlib import Path\\n\\nimport joblib\\nimport matplotlib\\nmatplotlib.use(\\'Agg\\')\\nimport matplotlib.pyplot as plt\\nimport numpy as np\\nimport pandas as pd\\nimport sklearn\\nimport xgboost\\nfrom sklearn.compose import ColumnTransformer\\nfrom sklearn.ensemble import RandomForestClassifier\\nfrom sklearn.impute import SimpleImputer\\nfrom sklearn.inspection import permutation_importance\\nfrom sklearn.linear_model import LogisticRegression\\nfrom sklearn.metrics import (average_precision_score, balanced_accuracy_score,\\n classification_report, confusion_matrix, ConfusionMatrixDisplay, f1_score,\\n fbeta_score, precision_score, recall_score, roc_auc_score, roc_curve)\\nfrom sklearn.model_selection import RandomizedSearchCV, StratifiedKFold, train_test_split\\nfrom sklearn.pipeline import Pipeline\\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\\nfrom xgboost import XGBClassifier\\n\\nSEED = 42\\nSPECS = {\\n \\'bank\\': {\\n \\'filename\\': \\'bank_customer_churn.csv\\', \\'target\\': \\'Exited\\',\\n \\'numeric\\': [\\'CreditScore\\',\\'Age\\',\\'Tenure\\',\\'Balance\\',\\'NumOfProducts\\',\\'HasCrCard\\',\\'IsActiveMember\\',\\'EstimatedSalary\\'],\\n \\'categorical\\': [\\'Geography\\'], \\'excluded\\': [\\'CustomerId\\',\\'Surname\\'],\\n \\'display\\': \\'Bank customer churn\\',\\n },\\n \\'telecom\\': {\\n \\'filename\\': \\'customer_churn_data.csv\\', \\'target\\': \\'Churn\\',\\n \\'numeric\\': [\\'SeniorCitizen\\',\\'tenure\\',\\'MonthlyCharges\\',\\'TotalCharges\\'],\\n \\'categorical\\': [\\'Partner\\',\\'Dependents\\',\\'PhoneService\\',\\'InternetService\\',\\'Contract\\',\\'PaymentMethod\\'],\\n \\'excluded\\': [\\'customerID\\'], \\'display\\': \\'Telecom customer churn\\',\\n },\\n}\\n\\ndef pyvalue(v):\\n if isinstance(v, np.generic):\\n return v.item()\\n return v\\n\\ndef load_dataset(path, spec):\\n if not path.is_file():\\n raise FileNotFoundError(f\\'Required dataset missing: {path}. Upload the original CSV in Colab.\\')\\n raw = pd.read_csv(path)\\n required = spec[\\'numeric\\'] + spec[\\'categorical\\'] + [spec[\\'target\\']]\\n missing = [c for c in required if c not in raw.columns]\\n if missing:\\n raise ValueError(f\\'{path.name}: missing columns {missing}. Found {list(raw.columns)}\\')\\n original_rows = len(raw)\\n frame = raw.drop_duplicates().copy()\\n # Never use identifiers, surnames, or the label as predictors.\\n X = frame[spec[\\'numeric\\'] + spec[\\'categorical\\']].copy()\\n for col in spec[\\'numeric\\']:\\n X[col] = pd.to_numeric(X[col], errors=\\'coerce\\')\\n for col in spec[\\'categorical\\']:\\n X[col] = X[col].astype(\\'string\\').str.strip().replace(\\'\\', pd.NA)\\n X[col] = X[col].astype(object).where(X[col].notna(), np.nan)\\n if spec[\\'target\\'] == \\'Churn\\':\\n y = frame[spec[\\'target\\']].astype(\\'string\\').str.strip().str.lower().map({\\'yes\\': 1, \\'no\\': 0})\\n else:\\n y = pd.to_numeric(frame[spec[\\'target\\']], errors=\\'coerce\\')\\n if y.isna().any() or not set(y.unique()).issubset({0, 1}):\\n raise ValueError(f\\'{path.name}: target must be fully labeled binary 0/1 or Yes/No\\')\\n y = y.astype(\\'int64\\')\\n if y.value_counts().min() < 20:\\n raise ValueError(f\\'{path.name}: insufficient positive/negative examples\\')\\n audit = {\\n \\'original_rows\\': int(original_rows),\\n \\'duplicate_rows_removed\\': int(original_rows-len(frame)),\\n \\'rows_after_deduplication\\': int(len(frame)),\\n \\'positive_rate\\': round(float(y.mean()), 5),\\n \\'feature_missing_counts\\': {col: int(X[col].isna().sum()) for col in X.columns},\\n \\'sha256_first12\\': hashlib.sha256(path.read_bytes()).hexdigest()[:12],\\n }\\n return X, y, audit\\n\\ndef preprocessing(spec):\\n numerical = Pipeline([(\\'impute\\', SimpleImputer(strategy=\\'median\\')),\\n (\\'scale\\', StandardScaler())])\\n categorical = Pipeline([(\\'impute\\', SimpleImputer(strategy=\\'most_frequent\\')),\\n (\\'onehot\\', OneHotEncoder(handle_unknown=\\'ignore\\', sparse_output=False))])\\n return ColumnTransformer([(\\'numeric\\', numerical, spec[\\'numeric\\']),\\n (\\'categorical\\', categorical, spec[\\'categorical\\'])],\\n remainder=\\'drop\\')\\n\\ndef score_proba(model, X):\\n return model.predict_proba(X)[:, 1]\\n\\ndef metrics(y, p, threshold):\\n pred = (p >= threshold).astype(int)\\n return {\\n \\'roc_auc\\': round(float(roc_auc_score(y, p)), 5),\\n \\'pr_auc\\': round(float(average_precision_score(y, p)), 5),\\n \\'balanced_accuracy\\': round(float(balanced_accuracy_score(y, pred)), 5),\\n \\'precision\\': round(float(precision_score(y, pred, zero_division=0)), 5),\\n \\'recall\\': round(float(recall_score(y, pred, zero_division=0)), 5),\\n \\'f1\\': round(float(f1_score(y, pred, zero_division=0)), 5),\\n \\'f2\\': round(float(fbeta_score(y, pred, beta=2, zero_division=0)), 5),\\n \\'confusion_matrix\\': confusion_matrix(y, pred, labels=[0, 1]).tolist(),\\n \\'classification_report\\': classification_report(y, pred, labels=[0, 1],\\n output_dict=True, zero_division=0),\\n }\\n\\ndef tune_threshold(y, p):\\n thresholds = np.round(np.arange(0.10, 0.901, 0.025), 3)\\n values = [fbeta_score(y, (p >= t).astype(int), beta=2, zero_division=0) for t in thresholds]\\n return float(thresholds[int(np.argmax(values))])\\n\\ndef plot_diagnostics(project, key, ytest, ptest, threshold, importance_df):\\n report_dir = project / \\'reports\\'\\n pred = (ptest >= threshold).astype(int)\\n fig, ax = plt.subplots(figsize=(5, 4))\\n ConfusionMatrixDisplay.from_predictions(ytest, pred, display_labels=[\\'Retained\\', \\'Churned\\'],\\n cmap=\\'Blues\\', colorbar=False, ax=ax)\\n ax.set_title(f\\'{key.title()} — untouched test data\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_confusion_matrix.png\\', dpi=140)\\n plt.close(fig)\\n fpr, tpr, _ = roc_curve(ytest, ptest)\\n fig, ax = plt.subplots(figsize=(5.5, 4))\\n ax.plot(fpr, tpr, label=f\\'ROC AUC = {roc_auc_score(ytest, ptest):.3f}\\')\\n ax.plot([0, 1], [0, 1], linestyle=\\'--\\', label=\\'Chance\\')\\n ax.set(xlabel=\\'False positive rate\\', ylabel=\\'True positive rate\\', title=f\\'{key.title()} — test ROC\\')\\n ax.legend(loc=\\'lower right\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_roc.png\\', dpi=140)\\n plt.close(fig)\\n top = importance_df.head(10).sort_values(\\'importance_mean\\')\\n fig, ax = plt.subplots(figsize=(7, 4.5))\\n ax.barh(top[\\'feature\\'], top[\\'importance_mean\\'])\\n ax.set(xlabel=\\'Change in validation PR AUC after shuffling\\', title=f\\'{key.title()} — permutation importance\\')\\n fig.tight_layout()\\n fig.savefig(report_dir / f\\'{key}_feature_importance.png\\', dpi=140)\\n plt.close(fig)\\n\\ndef train_one(project, source_dir, key, tuning_iterations):\\n spec = SPECS[key]\\n path = source_dir / spec[\\'filename\\']\\n X, y, audit = load_dataset(path, spec)\\n X_dev, X_test, y_dev, y_test = train_test_split(X, y, test_size=0.20,\\n random_state=SEED, stratify=y)\\n X_train, X_val, y_train, y_val = train_test_split(X_dev, y_dev, test_size=0.20,\\n random_state=SEED, stratify=y_dev)\\n minority_weight = float((y_train == 0).sum() / (y_train == 1).sum())\\n models = {\\n \\'Logistic regression\\': LogisticRegression(max_iter=1500, class_weight=\\'balanced\\', random_state=SEED),\\n \\'Random forest\\': RandomForestClassifier(n_estimators=170, min_samples_leaf=3,\\n class_weight=\\'balanced_subsample\\', n_jobs=-1, random_state=SEED),\\n }\\n candidates = {}\\n leaderboard = []\\n for name, estimator in models.items():\\n pipeline = Pipeline([(\\'preprocessor\\', preprocessing(spec)), (\\'model\\', estimator)])\\n pipeline.fit(X_train, y_train)\\n candidates[name] = pipeline\\n row = metrics(y_val, score_proba(pipeline, X_val), 0.5)\\n leaderboard.append({\\'dataset\\': key, \\'model\\': name, \\'validation_pr_auc\\': row[\\'pr_auc\\'],\\n \\'validation_roc_auc\\': row[\\'roc_auc\\'], \\'tuning\\': \\'none\\'})\\n xgb_pipe = Pipeline([(\\'preprocessor\\', preprocessing(spec)),\\n (\\'model\\', XGBClassifier(n_estimators=200, max_depth=3, learning_rate=0.05,\\n subsample=0.9, colsample_bytree=0.9, scale_pos_weight=minority_weight,\\n eval_metric=\\'logloss\\', tree_method=\\'hist\\', random_state=SEED, n_jobs=1))])\\n if tuning_iterations:\\n params = {\\n \\'model__n_estimators\\': [100, 180, 260],\\n \\'model__max_depth\\': [2, 3, 4],\\n \\'model__learning_rate\\': [0.025, 0.05, 0.1],\\n \\'model__subsample\\': [0.8, 1.0],\\n \\'model__colsample_bytree\\': [0.8, 1.0],\\n \\'model__min_child_weight\\': [1, 3],\\n }\\n search = RandomizedSearchCV(xgb_pipe, params, n_iter=tuning_iterations,\\n scoring=\\'average_precision\\', cv=StratifiedKFold(3, shuffle=True, random_state=SEED),\\n random_state=SEED, n_jobs=2, refit=True, verbose=0, error_score=\\'raise\\')\\n search.fit(X_train, y_train)\\n xgb_pipe = search.best_estimator_\\n print(f\\'[{key}] XGBoost best 3-fold training CV PR AUC: {search.best_score_:.4f}\\')\\n else:\\n xgb_pipe.fit(X_train, y_train)\\n candidates[\\'XGBoost\\'] = xgb_pipe\\n val_xgb = metrics(y_val, score_proba(xgb_pipe, X_val), 0.5)\\n leaderboard.append({\\'dataset\\': key, \\'model\\': \\'XGBoost\\',\\n \\'validation_pr_auc\\': val_xgb[\\'pr_auc\\'], \\'validation_roc_auc\\': val_xgb[\\'roc_auc\\'],\\n \\'tuning\\': f\\'{tuning_iterations}x3 CV\\' if tuning_iterations else \\'off\\'})\\n leaderboard.sort(key=lambda r: (r[\\'validation_pr_auc\\'], r[\\'validation_roc_auc\\']), reverse=True)\\n winner = leaderboard[0][\\'model\\']\\n champion = candidates[winner]\\n threshold = tune_threshold(y_val, score_proba(champion, X_val))\\n # The test partition has never been used to choose model, hyperparameters, or threshold.\\n test_scores = score_proba(champion, X_test)\\n test = metrics(y_test, test_scores, threshold)\\n # Deliberate launch-quality gate: near-chance classifiers remain research-only.\\n # Hold-out data are used for final model assessment, not training/selection.\\n reliable = (test[\\'roc_auc\\'] >= 0.60 and test[\\'pr_auc\\'] >= float(y_test.mean()) + 0.03)\\n deployment_status = \\'demonstration\\' if reliable else \\'research_only_low_signal\\'\\n if not reliable:\\n print(f\\'[{key}] WARNING: near-chance model; API will not issue actionable churn labels.\\')\\n val = metrics(y_val, score_proba(champion, X_val), threshold)\\n defaults = {c: float(X_train[c].median()) for c in spec[\\'numeric\\']}\\n categories = {}\\n for col in spec[\\'categorical\\']:\\n vals = X_train[col].dropna().astype(str)\\n categories[col] = sorted(vals.unique().tolist())\\n defaults[col] = str(vals.mode().iloc[0])\\n importance = permutation_importance(champion, X_val, y_val, scoring=\\'average_precision\\',\\n n_repeats=3, random_state=SEED, n_jobs=1)\\n importance_df = pd.DataFrame({\\'feature\\': spec[\\'numeric\\'] + spec[\\'categorical\\'],\\n \\'importance_mean\\': importance.importances_mean, \\'importance_std\\': importance.importances_std})\\n importance_df = importance_df.sort_values(\\'importance_mean\\', ascending=False)\\n importance_df.to_csv(project / \\'reports\\' / f\\'{key}_feature_importance.csv\\', index=False)\\n plot_diagnostics(project, key, y_test, test_scores, threshold, importance_df)\\n bundle = {\\'pipeline\\': champion, \\'threshold\\': threshold,\\n \\'numeric\\': spec[\\'numeric\\'], \\'categorical\\': spec[\\'categorical\\'], \\'features\\': spec[\\'numeric\\'] + spec[\\'categorical\\']}\\n path_model = project / \\'models\\' / f\\'{key}_churn.joblib\\'\\n joblib.dump(bundle, path_model, compress=3)\\n reloaded = joblib.load(path_model)\\n assert np.allclose(reloaded[\\'pipeline\\'].predict_proba(X_test.iloc[:5])[:,1], test_scores[:5])\\n metadata = {\\n \\'label\\': spec[\\'display\\'], \\'selected_model\\': winner,\\n \\'numeric\\': spec[\\'numeric\\'], \\'categorical\\': spec[\\'categorical\\'],\\n \\'features\\': spec[\\'numeric\\'] + spec[\\'categorical\\'],\\n \\'excluded_columns\\': spec[\\'excluded\\'], \\'categorical_choices\\': categories,\\n \\'defaults\\': defaults, \\'threshold\\': threshold,\\n \\'training_rows\\': len(X_train), \\'validation_rows\\': len(X_val), \\'test_rows\\': len(X_test),\\n \\'audit\\': audit, \\'validation_metrics\\': val, \\'test_metrics\\': test,\\n \\'deployment_status\\': deployment_status,\\n \\'model_size_mb\\': round(path_model.stat().st_size / 1024**2, 3),\\n \\'validation_model_comparison\\': leaderboard,\\n }\\n print(f\\'[{key}] Winner: {winner}, threshold: {threshold:.3f}, \\'\\n f\\'test PR AUC: {test[\"pr_auc\"]:.4f}, ROC AUC: {test[\"roc_auc\"]:.4f}, recall: {test[\"recall\"]:.4f}\\')\\n return metadata, leaderboard\\n\\ndef main(data_dir=\\'.\\', project_dir=\\'.\\', tuning_iterations=6):\\n project = Path(project_dir).resolve()\\n data_dir = Path(data_dir).resolve()\\n (project / \\'models\\').mkdir(parents=True, exist_ok=True)\\n (project / \\'reports\\').mkdir(parents=True, exist_ok=True)\\n results, leaderboard = {}, []\\n for key in SPECS:\\n print(f\\'\\\\n=== TRAINING {key.upper()} ===\\')\\n results[key], comparison = train_one(project, data_dir, key, tuning_iterations)\\n leaderboard.extend(comparison)\\n summary = {\\'generated_utc\\': datetime.now(timezone.utc).isoformat(),\\n \\'random_seed\\': SEED, \\'sklearn_version\\': sklearn.__version__,\\n \\'xgboost_version\\': xgboost.__version__, \\'python_version\\': sys.version.split()[0], \\'datasets\\': results,\\n \\'decision_rule\\': \\'maximize validation F2 (recall-weighted), after selecting by validation PR AUC\\',\\n \\'probability_warning\\': \\'Scores are uncalibrated model probabilities; not validated as true churn probabilities.\\'}\\n (project / \\'model_metadata.json\\').write_text(json.dumps(summary, indent=2, default=pyvalue), encoding=\\'utf-8\\')\\n pd.DataFrame(leaderboard).to_csv(project / \\'reports\\' / \\'model_comparison.csv\\', index=False)\\n print(\\'\\\\nSaved trained pipelines, evaluation, charts, and audit metadata to\\', project)\\n return summary\\n\\nif __name__ == \\'__main__\\':\\n parser = argparse.ArgumentParser()\\n parser.add_argument(\\'--data-dir\\', default=\\'.\\')\\n parser.add_argument(\\'--project-dir\\', default=\\'.\\')\\n parser.add_argument(\\'--tuning-iterations\\', type=int, default=6)\\n args = parser.parse_args()\\n main(args.data_dir, args.project_dir, args.tuning_iterations)\\n'\n",
"Path(\"train.py\").write_text(TRAIN_SOURCE, encoding=\"utf-8\")\n",
"import sklearn, xgboost\n",
"assert (sklearn.__version__,xgboost.__version__)==(\"1.8.0\",\"3.1.3\"), \"Restart Colab runtime after install and rerun cells\"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "097cd908",
"metadata": {},
"outputs": [],
"source": [
"import importlib.util\n",
"spec=importlib.util.spec_from_file_location(\"churn_training\", \"train.py\")\n",
"module=importlib.util.module_from_spec(spec)\n",
"spec.loader.exec_module(module)\n",
"results=module.main(data_dir=DATA,project_dir=Path(\"/content/churn-intelligence-lab\"),tuning_iterations=6)\n"
]
},
{
"cell_type": "markdown",
"id": "185df73a",
"metadata": {},
"source": [
"## Compare model performance and diagnostics\n",
"The quality gate marks a near-chance classifier as research-only. The test partition is independent of validation-based selection."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ffcd1d62",
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import display, Image\n",
"import pandas as pd\n",
"P=Path(\"/content/churn-intelligence-lab\")\n",
"display(pd.read_csv(P/\"reports\"/\"model_comparison.csv\"))\n",
"for segment in (\"bank\",\"telecom\"):\n",
" print(segment.upper(),results[\"datasets\"][segment][\"test_metrics\"])\n",
" display(Image(filename=str(P/\"reports\"/f\"{segment}_roc.png\"),width=550))\n",
" display(Image(filename=str(P/\"reports\"/f\"{segment}_feature_importance.png\"),width=550))"
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|