core/linear_regression.py

from __future__ import annotations

from typing import List, Optional, Sequence, Tuple

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
import statsmodels.formula.api as smf
from matplotlib.figure import Figure


def plot_simple_regression(

    *,

    data: pd.DataFrame,

    x: str,

    y: str,

    intercept: bool,

    use_formula: bool,

    formula_latex: str,

    model: sm.regression.linear_model.RegressionResultsWrapper,

    alpha: float,

    show_ci: bool,

    show_pi: bool,

    fit_to_obs: bool,

    x_vector: Optional[Sequence[float]],

) -> Figure:
    """

    Replicates the behaviour of PlotSimpleRegression in the monolith.



    Parameters

    ----------

    data:

        DataFrame containing at least the columns `x` and `y` (after dropping NaNs).

    x, y:

        Names of the predictor and response.

    intercept:

        Whether an explicit intercept column was included when fitting the model

        (ignored if `use_formula` is True, since the formula controls the intercept).

    use_formula:

        Whether the model was fit via a formula.

    formula_latex:

        Optional LaTeX representation of the model, used for the title.

    model:

        Fitted statsmodels OLS / OLS-like result.

    alpha:

        Significance level used for confidence / prediction intervals (e.g. 0.05).

    show_ci, show_pi:

        Whether to plot confidence / prediction intervals.

    fit_to_obs:

        If True, predictions are evaluated on the observed X values (sorted).

        Otherwise, predictions are evaluated on `x_vector`, which must be

        a 1-D sequence of x values.

    x_vector:

        Grid of x values to use when `fit_to_obs` is False.



    Returns

    -------

    matplotlib.figure.Figure

    """
    # Prepare prediction input
    if fit_to_obs:
        work = data[[x, y]].dropna().copy().sort_values(x).reset_index(drop=True)
        x_plot = work[x].to_numpy()
        X_pred = work[[x]]
    else:
        if x_vector is None:
            raise ValueError("x_vector must be provided when 'fit_to_obs' is False.")
        x_arr = np.asarray(x_vector)
        x_plot = x_arr
        X_pred = pd.DataFrame({x: x_arr})

    # Add intercept if needed (only relevant for non-formula fits)
    if not use_formula and intercept:
        X_pred = sm.add_constant(X_pred)

    # Get prediction intervals from statsmodels
    pred_table = model.get_prediction(X_pred).summary_frame(alpha=alpha)

    # --- Plotting ---
    plt.style.use("seaborn-v0_8-whitegrid")
    fig, ax = plt.subplots(figsize=(8, 5.5))

    # Scatter plot of data
    sns.scatterplot(
        data=data,
        x=x,
        y=y,
        ax=ax,
        s=50,
        edgecolor="black",
        linewidth=0.5,
        zorder=3,
        label="Data",
        alpha=0.5,
    )

    # Regression line
    ax.plot(x_plot, pred_table["mean"], color="royalblue", linewidth=2, label="Prediction")

    # Confidence interval for mean
    if show_ci:
        ax.fill_between(
            x_plot,
            pred_table["mean_ci_lower"],
            pred_table["mean_ci_upper"],
            color="pink",
            alpha=0.5,
            label="Confidence Interval (mean)",
        )

    # Prediction interval for new observations
    if show_pi:
        ax.fill_between(
            x_plot,
            pred_table["obs_ci_lower"],
            pred_table["obs_ci_upper"],
            color="mediumpurple",
            alpha=0.4,
            label="Prediction Interval (new obs)",
        )

    # Highlight extrapolation region when using x_vector
    if not fit_to_obs:
        xmin, xmax = data[x].min(), data[x].max()
        ax.axvspan(x_plot[0], xmin, color="gray", alpha=0.1)
        ax.axvspan(xmax, x_plot[-1], color="gray", alpha=0.1)

    # Title
    if use_formula and formula_latex:
        ax.set_title(f"Linear Regression: ${formula_latex}$", fontsize=14)
    else:
        ax.set_title(f"Linear Regression: {y} ~ {x}", fontsize=14)

    # R-squared annotation
    r2 = getattr(model, "rsquared", None)
    if r2 is not None:
        ax.text(
            0.05,
            0.95,
            f"$R^2 = {r2:.3f}$",
            transform=ax.transAxes,
            ha="left",
            va="top",
            fontsize=11,
            bbox=dict(boxstyle="round", facecolor="white", alpha=0.8),
        )

    ax.set_xlabel(x, fontsize=12)
    ax.set_ylabel(y, fontsize=12)

    # Deduplicate legend entries
    handles, labels = ax.get_legend_handles_labels()
    by_label = dict(zip(labels, handles))
    ax.legend(by_label.values(), by_label.keys(), frameon=False)

    ax.grid(True, linestyle="--", alpha=0.3)
    plt.tight_layout()

    return fig


def plot_observed_vs_predicted(

    *,

    data: pd.DataFrame,

    y: str,

    model: sm.regression.linear_model.RegressionResultsWrapper,

    alpha: float = 0.05,

) -> Figure:
    """

    Replicates PlotCompareYHatY from the monolith.

    """
    plt.style.use("seaborn-v0_8-whitegrid")

    pred_table = model.get_prediction().summary_frame(alpha=alpha)
    y_true = data[y]
    y_pred = pred_table["mean"]
    y_err = pred_table["obs_ci_upper"] - y_pred

    residuals = y_true - y_pred

    fig, ax = plt.subplots(figsize=(7.5, 5.5))

    # Scatter of observed vs predicted, coloured by |residual|
    sc = ax.scatter(
        y_true,
        y_pred,
        c=np.abs(residuals),
        cmap="Reds",
        edgecolor="black",
        alpha=0.6,
        s=60,
        label="Predicted vs Observed",
        zorder=3,
    )

    # Error bars using prediction interval width
    ax.errorbar(
        y_true,
        y_pred,
        yerr=y_err,
        fmt="none",
        ecolor="gray",
        elinewidth=1,
        alpha=0.4,
        capsize=3,
        zorder=1,
    )

    # Reference 45° line
    min_val = min(y_true.min(), y_pred.min())
    max_val = max(y_true.max(), y_pred.max())
    buffer = 0.05 * (max_val - min_val)
    ax.plot(
        [min_val, max_val],
        [min_val, max_val],
        "r--",
        label="Perfect Fit",
        zorder=2,
    )
    ax.set_xlim(min_val - buffer, max_val + buffer)
    ax.set_ylim(min_val - buffer, max_val + buffer)

    ax.set_title("Observed vs Predicted", fontsize=14)
    ax.set_xlabel(f"Observed {y}", fontsize=12)
    ax.set_ylabel(f"Predicted {y}", fontsize=12)

    r2 = getattr(model, "rsquared", None)
    if r2 is not None:
        ax.text(
            0.05,
            0.95,
            f"$R^2 = {r2:.3f}$",
            transform=ax.transAxes,
            ha="left",
            va="top",
            fontsize=11,
            bbox=dict(boxstyle="round", facecolor="white", alpha=0.7),
        )

    cbar = plt.colorbar(sc, ax=ax)
    cbar.set_label("|Residual|", rotation=270, labelpad=15)

    handles, labels = ax.get_legend_handles_labels()
    ax.legend(handles, labels, frameon=False)
    ax.grid(True, linestyle="--", alpha=0.3)
    plt.tight_layout()

    return fig


def run_linear_regression(

    *,

    df: pd.DataFrame,

    formula_check: bool,

    formula_text: str,

    formula_latex: str,

    dependent_var: str,

    independent_vars: Sequence[str],

    alpha: float,

    intercept: bool,

    create_graph: bool,

    graph_type: str,

    show_ci: bool,

    show_pi: bool,

    fit_to_obs: bool,

    x_vector: Optional[Sequence[float]],

) -> Tuple[str, pd.DataFrame, Optional[Figure]]:
    """

    Fit a linear regression model and optionally create a diagnostic plot.



    This is a stats-only function: it does not depend on Gradio or any UI

    objects. It mirrors the behaviour of the monolithic `linear_regression`

    function, but leaves error handling and rounding to the caller.



    Returns

    -------

    summary_html:

        HTML representation of statsmodels' summary2 table.

    params_table:

        Coefficient table (second table from summary2) as a DataFrame

        with a 'Variable' column (unrounded).

    fig:

        Matplotlib figure for the requested graph, or None if

        create_graph is False.

    """
    if df is None or df.empty:
        raise ValueError("No dataset loaded.")

    if dependent_var not in df.columns:
        raise ValueError("Invalid dependent variable selection.")

    for col in independent_vars:
        if col not in df.columns:
            raise ValueError(f"Invalid independent variable: {col!r}")

    if len(independent_vars) == 0:
        raise ValueError("At least one independent variable is required.")

    # Drop rows with missing values in the relevant columns
    cols = [dependent_var] + list(independent_vars)
    data = df[cols].dropna()

    if data.empty:
        raise ValueError("No valid rows remaining after dropping missing values.")

    y = data[dependent_var]
    X = data[list(independent_vars)]

    # Fit model
    if formula_check:
        # When using a formula, the intercept is fully controlled by the formula.
        if not formula_text:
            raise ValueError("Formula is enabled but no formula text was provided.")
        model = smf.ols(data=data, formula=formula_text).fit()
        use_formula = True
    else:
        if intercept:
            X = sm.add_constant(X)
        model = sm.OLS(y, X).fit()
        use_formula = False

    # Build summary objects
    summary = model.summary2(alpha=alpha)
    summary_html = summary.as_html()

    # Coefficient table is the second table in summary2
    params_table = summary.tables[1].reset_index().rename(columns={"index": "Variable"})

    fig: Optional[Figure] = None

    if create_graph:
        if graph_type == "Simple Regression":
            if len(independent_vars) != 1:
                raise ValueError(
                    "Simple Regression graph is only available with exactly one "
                    "independent variable."
                )

            x_col = independent_vars[0]
            fig = plot_simple_regression(
                data=data,
                x=x_col,
                y=dependent_var,
                intercept=intercept,
                use_formula=use_formula,
                formula_latex=formula_latex,
                model=model,
                alpha=alpha,
                show_ci=show_ci,
                show_pi=show_pi,
                fit_to_obs=fit_to_obs,
                x_vector=x_vector,
            )
        elif graph_type == "Observed vs Predicted":
            fig = plot_observed_vs_predicted(
                data=data,
                y=dependent_var,
                model=model,
                alpha=alpha,
            )
        else:
            raise ValueError(f"Unknown graph type: {graph_type!r}")

    return summary_html, params_table, fig