File size: 9,857 Bytes
f41fa00 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 |
import sys
import time
import numpy as np
import pandas as pd
import joblib
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder, PolynomialFeatures
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
def select_target_column(df, target_column=None):
# Displays available columns/dtypes and validates (or interactively
# selects) a numeric target column. Not tied to any one dataset's schema.
print("Available columns:")
for c in df.columns:
print(f" {c}: {df[c].dtype}")
if target_column is None:
try:
interactive = sys.stdin.isatty()
except Exception:
interactive = False
if interactive:
target_column = input("Enter the target column name for regression: ").strip()
else:
numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
if not numeric_cols:
raise ValueError("No numeric columns available to use as a regression target.")
target_column = numeric_cols[-1]
print(f"No target_column specified and no interactive input available; "
f"auto-selected the last numeric column as target: '{target_column}'")
if target_column not in df.columns:
raise ValueError(f"Target column '{target_column}' not found. Available columns: {list(df.columns)}")
if not pd.api.types.is_numeric_dtype(df[target_column]):
raise ValueError(f"Target column '{target_column}' is not numeric; regression requires a numeric target.")
print(f"Using target column: '{target_column}'")
return target_column
def detect_datetime_columns(df, exclude):
cols = []
for c in df.columns:
if c in exclude:
continue
if pd.api.types.is_datetime64_any_dtype(df[c]):
cols.append(c)
elif df[c].dtype == object:
parsed = pd.to_datetime(df[c], errors="coerce")
if parsed.notna().mean() > 0.95:
cols.append(c)
return cols
def engineer_datetime_features(df, datetime_columns):
# Any detected datetime column becomes numeric calendar features
# (hour/day-of-week/month/year) instead of being used raw, then is
# dropped -- generic for any dataset that happens to carry a timestamp.
df = df.copy()
new_cols = []
for c in datetime_columns:
dt = pd.to_datetime(df[c], errors="coerce")
for part, values in [("hour", dt.dt.hour), ("dayofweek", dt.dt.dayofweek), ("month", dt.dt.month), ("year", dt.dt.year)]:
col_name = f"{c}__{part}"
df[col_name] = values
new_cols.append(col_name)
df = df.drop(columns=datetime_columns)
return df, new_cols
def detect_feature_columns(df, target_column):
exclude = {target_column}
datetime_columns = detect_datetime_columns(df, exclude)
if datetime_columns:
df_engineered, _ = engineer_datetime_features(df, datetime_columns)
else:
df_engineered = df
numeric_features = [c for c in df_engineered.columns if c != target_column and pd.api.types.is_numeric_dtype(df_engineered[c])]
categorical_features = [
c for c in df_engineered.columns
if c != target_column and c not in numeric_features and not pd.api.types.is_datetime64_any_dtype(df_engineered[c])
]
return df_engineered, numeric_features, categorical_features, datetime_columns
def build_preprocessor(numeric_features, categorical_features):
transformers = []
if numeric_features:
transformers.append((
"numeric",
Pipeline([("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]),
numeric_features,
))
if categorical_features:
transformers.append((
"categorical",
Pipeline([("imputer", SimpleImputer(strategy="most_frequent")),
("onehot", OneHotEncoder(handle_unknown="ignore"))]),
categorical_features,
))
return ColumnTransformer(transformers=transformers, remainder="drop")
class GenericRegressionModel:
# Dataset-agnostic regression pipeline: give it ANY tabular dataframe and
# a target column name; it detects numeric/categorical/datetime features,
# builds preprocessing, and fits the wrapped estimator. Nothing here is
# specific to any one dataset's column names, shapes, or coefficients.
def __init__(self, estimator_factory, model_key, model_title, feature_columns=None, polynomial_degree=None):
self.estimator_factory = estimator_factory
self.model_key = model_key
self.model_title = model_title
self.forced_feature_columns = feature_columns
self.polynomial_degree = polynomial_degree
self.pipeline = None
self.target_column = None
self.datetime_columns = None
self.numeric_features = None
self.categorical_features = None
self.feature_columns = None
def fit(self, df, target_column):
if target_column not in df.columns:
raise ValueError(f"Target column '{target_column}' not found. Available columns: {list(df.columns)}")
if not pd.api.types.is_numeric_dtype(df[target_column]):
raise ValueError(f"Target column '{target_column}' is not numeric; regression requires a numeric target.")
self.target_column = target_column
df_engineered, numeric_features, categorical_features, datetime_columns = detect_feature_columns(df, target_column)
self.datetime_columns = datetime_columns
if self.forced_feature_columns is not None:
allowed = set(self.forced_feature_columns)
numeric_features = [c for c in numeric_features if c in allowed]
categorical_features = [c for c in categorical_features if c in allowed]
if not numeric_features and not categorical_features:
raise ValueError("No usable feature columns detected after excluding the target.")
self.numeric_features = numeric_features
self.categorical_features = categorical_features
self.feature_columns = numeric_features + categorical_features
preprocessor = build_preprocessor(numeric_features, categorical_features)
steps = [("preprocess", preprocessor)]
if self.polynomial_degree:
steps.append(("poly", PolynomialFeatures(degree=self.polynomial_degree, include_bias=False)))
steps.append(("model", self.estimator_factory()))
self.pipeline = Pipeline(steps)
X = df_engineered[self.feature_columns]
y = df_engineered[target_column]
self.pipeline.fit(X, y)
return self
def predict(self, new_df):
if self.pipeline is None:
raise RuntimeError("Model has not been fit yet.")
df_engineered = new_df
if self.datetime_columns:
present = [c for c in self.datetime_columns if c in new_df.columns]
if present:
df_engineered, _ = engineer_datetime_features(new_df, present)
missing = [c for c in self.feature_columns if c not in df_engineered.columns]
if missing:
raise ValueError(f"Input is missing expected feature columns: {missing}")
return self.pipeline.predict(df_engineered[self.feature_columns])
def save(self, path):
joblib.dump(self, path)
@staticmethod
def load(path):
return joblib.load(path)
def _generic_train_and_predict(data, target_column, estimator_factory, model_key, model_title,
feature_columns=None, polynomial_degree=None, test_size=0.2,
random_state=42, max_train_samples=None):
# Reusable training/evaluation engine every model-specific train_and_predict()
# below delegates to. `data` and `target_column` fully determine behavior --
# nothing here references any dataset by name.
model = GenericRegressionModel(estimator_factory, model_key, model_title, feature_columns, polynomial_degree)
train_df, test_df = train_test_split(data, test_size=test_size, random_state=random_state, shuffle=True)
if max_train_samples and len(train_df) > max_train_samples:
train_df = train_df.sample(n=max_train_samples, random_state=random_state)
start_time = time.time()
model.fit(train_df, target_column)
training_time = time.time() - start_time
predictions = model.predict(test_df)
y_test = test_df[target_column].values
return {
"model": model, "predictions": predictions, "y_test": y_test,
"training_time": training_time, "test_df": test_df, "train_df": train_df,
}
# --- Model-specific configuration (everything above this line is fully generic) ---
from sklearn.linear_model import LinearRegression
MODEL_KEY = "linear-regression"
MODEL_TITLE = "Simple Linear Regression"
POLYNOMIAL_DEGREE = None
def _estimator_factory():
return LinearRegression()
def train_and_predict(data, target_column, feature_columns=None, test_size=0.2, random_state=42, max_train_samples=None):
# Matches: result = train_and_predict(data=df, target_column="target")
return _generic_train_and_predict(
data, target_column, estimator_factory=_estimator_factory, model_key=MODEL_KEY, model_title=MODEL_TITLE,
feature_columns=feature_columns, polynomial_degree=POLYNOMIAL_DEGREE, test_size=test_size,
random_state=random_state, max_train_samples=max_train_samples,
)
|