app.py

import gradio as gr
import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.metrics import log_loss

from scipy.special import expit

theme = gr.themes.Monochrome(
    primary_hue="indigo",
    secondary_hue="blue",
    neutral_hue="slate",
)
model_card = f"""
## Description

The **Out-of-bag (OOB)** method is a useful technique for estimating the optimal number of boosting iterations.
This method is similar to cross-validation, but it does not require repeated model fitting and can be computed on-the-fly. 
**OOB** estimates are only applicable to Stochastic Gradient Boosting (i.e., subsample < 1.0). They are calculated from the improvement in loss based on examples not included in the bootstrap sample (i.e., out-of-bag examples). 
The **OOB** estimator provides a conservative estimate of the true test loss but is still a reasonable approximation for a small number of trees. 
In this demonstration, a **GradientBoostingClassifier** model is trained on a simulation dataset, and the loss of the training set, test set, and OOB set are displayed in the figure.
This information allows you to determine the level of generalization of your trained model on the simulation dataset.
You can play around with ``number of samples``,``number of splits fold``, ``random seed``and ``number of estimation step``

## Dataset

Simulation data
"""

def do_train(n_samples, n_splits, random_seed, n_estimators):
    # Generate data (adapted from G. Ridgeway's gbm example)
    random_state = np.random.RandomState(random_seed)
    x1 = random_state.uniform(size=n_samples)
    x2 = random_state.uniform(size=n_samples)
    x3 = random_state.randint(0, 4, size=n_samples)

    p = expit(np.sin(3 * x1) - 4 * x2 + x3)
    y = random_state.binomial(1, p, size=n_samples)

    X = np.c_[x1, x2, x3]

    X = X.astype(np.float32)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=random_seed)

    # Fit classifier with out-of-bag estimates
    params = {
        "n_estimators": n_estimators,
        "max_depth": 3,
        "subsample": 0.5,
        "learning_rate": 0.01,
        "min_samples_leaf": 1,
        "random_state": random_seed,
    }
    clf = GradientBoostingClassifier(**params)

    clf.fit(X_train, y_train)
    train_acc = clf.score(X_train, y_train)
    test_acc = clf.score(X_test, y_test)
    text = f"Train set accuracy: {train_acc*100:.2f}%. Test set accuracy: {test_acc*100:.2f}%"
    n_estimators = params["n_estimators"]
    x = np.arange(n_estimators) + 1

    def heldout_score(clf, X_test, y_test):
        """compute deviance scores on ``X_test`` and ``y_test``."""
        score = np.zeros((n_estimators,), dtype=np.float64)
        for i, y_proba in enumerate(clf.staged_predict_proba(X_test)):
            score[i] = 2 * log_loss(y_test, y_proba[:, 1])
        return score

    def cv_estimate(n_splits):
        cv = KFold(n_splits=n_splits)
        cv_clf = GradientBoostingClassifier(**params)
        val_scores = np.zeros((n_estimators,), dtype=np.float64)
        for train, test in cv.split(X_train, y_train):
            cv_clf.fit(X_train[train], y_train[train])
            val_scores += heldout_score(cv_clf, X_train[test], y_train[test])
        val_scores /= n_splits
        return val_scores

    # Estimate best n_splits using cross-validation
    cv_score = cv_estimate(n_splits)

    # Compute best n_splits for test data
    test_score = heldout_score(clf, X_test, y_test)

    # negative cumulative sum of oob improvements
    cumsum = -np.cumsum(clf.oob_improvement_)

    # min loss according to OOB
    oob_best_iter = x[np.argmin(cumsum)]

    # min loss according to test (normalize such that first loss is 0)
    test_score -= test_score[0]
    test_best_iter = x[np.argmin(test_score)]

    # min loss according to cv (normalize such that first loss is 0)
    cv_score -= cv_score[0]
    cv_best_iter = x[np.argmin(cv_score)]

    # color brew for the three curves
    oob_color = list(map(lambda x: x / 256.0, (190, 174, 212)))
    test_color = list(map(lambda x: x / 256.0, (127, 201, 127)))
    cv_color = list(map(lambda x: x / 256.0, (253, 192, 134)))

    # line type for the three curves
    oob_line = "dashed"
    test_line = "solid"
    cv_line = "dashdot"

    # plot curves and vertical lines for best iterations
    fig, ax = plt.subplots(figsize=(8, 6))
    ax.plot(x, cumsum, label="OOB loss", color=oob_color, linestyle=oob_line)
    ax.plot(x, test_score, label="Test loss", color=test_color, linestyle=test_line)
    ax.plot(x, cv_score, label="CV loss", color=cv_color, linestyle=cv_line)
    ax.axvline(x=oob_best_iter, color=oob_color, linestyle=oob_line)
    ax.axvline(x=test_best_iter, color=test_color, linestyle=test_line)
    ax.axvline(x=cv_best_iter, color=cv_color, linestyle=cv_line)

    # add three vertical lines to xticks
    xticks = plt.xticks()
    xticks_pos = np.array(
        xticks[0].tolist() + [oob_best_iter, cv_best_iter, test_best_iter]
    )
    xticks_label = np.array(list(map(lambda t: int(t), xticks[0])) + ["OOB", "CV", "Test"])
    ind = np.argsort(xticks_pos)
    xticks_pos = xticks_pos[ind]
    xticks_label = xticks_label[ind]
    ax.set_xticks(xticks_pos, xticks_label, rotation=90)

    ax.legend(loc="upper center")
    ax.set_ylabel("normalized loss")
    ax.set_xlabel("number of iterations")
    return fig, text


with gr.Blocks(theme=theme) as demo:
    gr.Markdown('''
            <div>
            <h1 style='text-align: center'>Gradient Boosting Out-of-Bag estimates</h1>
            </div>
        ''')
    gr.Markdown(model_card)
    gr.Markdown("Author: <a href=\"https://huggingface.co/vumichien\">Vu Minh Chien</a>. Based on the example from <a href=\"https://scikit-learn.org/stable/auto_examples/ensemble/plot_gradient_boosting_oob.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-oob-py\">scikit-learn</a>")
    n_samples = gr.Slider(minimum=500, maximum=5000, step=500, value=500, label="Number of samples")
    n_splits = gr.Slider(minimum=2, maximum=10, step=1, value=3, label="Number of cross validation folds")
    random_seed = gr.Slider(minimum=0, maximum=2000, step=1, value=0, label="Random seed")
    n_estimators = gr.Slider(minimum=500, maximum=2000, step=100, value=500, label="Number of step")

    with gr.Row():
        with gr.Column():
            plot = gr.Plot()
        with gr.Column():
            result = gr.Textbox(label="Resusts")
        
    n_samples.change(fn=do_train, inputs=[n_samples, n_splits, random_seed, n_estimators], outputs=[plot, result])
    n_splits.change(fn=do_train, inputs=[n_samples, n_splits, random_seed, n_estimators], outputs=[plot, result])
    random_seed.change(fn=do_train, inputs=[n_samples, n_splits, random_seed, n_estimators], outputs=[plot, result])
    n_estimators.change(fn=do_train, inputs=[n_samples, n_splits, random_seed, n_estimators], outputs=[plot, result])

demo.launch()