File size: 9,857 Bytes
f41fa00
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226

import sys
import time
import numpy as np
import pandas as pd
import joblib
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder, PolynomialFeatures
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split


def select_target_column(df, target_column=None):
    # Displays available columns/dtypes and validates (or interactively
    # selects) a numeric target column. Not tied to any one dataset's schema.
    print("Available columns:")
    for c in df.columns:
        print(f"  {c}: {df[c].dtype}")

    if target_column is None:
        try:
            interactive = sys.stdin.isatty()
        except Exception:
            interactive = False
        if interactive:
            target_column = input("Enter the target column name for regression: ").strip()
        else:
            numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
            if not numeric_cols:
                raise ValueError("No numeric columns available to use as a regression target.")
            target_column = numeric_cols[-1]
            print(f"No target_column specified and no interactive input available; "
                  f"auto-selected the last numeric column as target: '{target_column}'")

    if target_column not in df.columns:
        raise ValueError(f"Target column '{target_column}' not found. Available columns: {list(df.columns)}")
    if not pd.api.types.is_numeric_dtype(df[target_column]):
        raise ValueError(f"Target column '{target_column}' is not numeric; regression requires a numeric target.")

    print(f"Using target column: '{target_column}'")
    return target_column


def detect_datetime_columns(df, exclude):
    cols = []
    for c in df.columns:
        if c in exclude:
            continue
        if pd.api.types.is_datetime64_any_dtype(df[c]):
            cols.append(c)
        elif df[c].dtype == object:
            parsed = pd.to_datetime(df[c], errors="coerce")
            if parsed.notna().mean() > 0.95:
                cols.append(c)
    return cols


def engineer_datetime_features(df, datetime_columns):
    # Any detected datetime column becomes numeric calendar features
    # (hour/day-of-week/month/year) instead of being used raw, then is
    # dropped -- generic for any dataset that happens to carry a timestamp.
    df = df.copy()
    new_cols = []
    for c in datetime_columns:
        dt = pd.to_datetime(df[c], errors="coerce")
        for part, values in [("hour", dt.dt.hour), ("dayofweek", dt.dt.dayofweek), ("month", dt.dt.month), ("year", dt.dt.year)]:
            col_name = f"{c}__{part}"
            df[col_name] = values
            new_cols.append(col_name)
    df = df.drop(columns=datetime_columns)
    return df, new_cols


def detect_feature_columns(df, target_column):
    exclude = {target_column}
    datetime_columns = detect_datetime_columns(df, exclude)
    if datetime_columns:
        df_engineered, _ = engineer_datetime_features(df, datetime_columns)
    else:
        df_engineered = df
    numeric_features = [c for c in df_engineered.columns if c != target_column and pd.api.types.is_numeric_dtype(df_engineered[c])]
    categorical_features = [
        c for c in df_engineered.columns
        if c != target_column and c not in numeric_features and not pd.api.types.is_datetime64_any_dtype(df_engineered[c])
    ]
    return df_engineered, numeric_features, categorical_features, datetime_columns


def build_preprocessor(numeric_features, categorical_features):
    transformers = []
    if numeric_features:
        transformers.append((
            "numeric",
            Pipeline([("imputer", SimpleImputer(strategy="median")), ("scaler", StandardScaler())]),
            numeric_features,
        ))
    if categorical_features:
        transformers.append((
            "categorical",
            Pipeline([("imputer", SimpleImputer(strategy="most_frequent")),
                      ("onehot", OneHotEncoder(handle_unknown="ignore"))]),
            categorical_features,
        ))
    return ColumnTransformer(transformers=transformers, remainder="drop")


class GenericRegressionModel:
    # Dataset-agnostic regression pipeline: give it ANY tabular dataframe and
    # a target column name; it detects numeric/categorical/datetime features,
    # builds preprocessing, and fits the wrapped estimator. Nothing here is
    # specific to any one dataset's column names, shapes, or coefficients.
    def __init__(self, estimator_factory, model_key, model_title, feature_columns=None, polynomial_degree=None):
        self.estimator_factory = estimator_factory
        self.model_key = model_key
        self.model_title = model_title
        self.forced_feature_columns = feature_columns
        self.polynomial_degree = polynomial_degree
        self.pipeline = None
        self.target_column = None
        self.datetime_columns = None
        self.numeric_features = None
        self.categorical_features = None
        self.feature_columns = None

    def fit(self, df, target_column):
        if target_column not in df.columns:
            raise ValueError(f"Target column '{target_column}' not found. Available columns: {list(df.columns)}")
        if not pd.api.types.is_numeric_dtype(df[target_column]):
            raise ValueError(f"Target column '{target_column}' is not numeric; regression requires a numeric target.")

        self.target_column = target_column
        df_engineered, numeric_features, categorical_features, datetime_columns = detect_feature_columns(df, target_column)
        self.datetime_columns = datetime_columns

        if self.forced_feature_columns is not None:
            allowed = set(self.forced_feature_columns)
            numeric_features = [c for c in numeric_features if c in allowed]
            categorical_features = [c for c in categorical_features if c in allowed]

        if not numeric_features and not categorical_features:
            raise ValueError("No usable feature columns detected after excluding the target.")

        self.numeric_features = numeric_features
        self.categorical_features = categorical_features
        self.feature_columns = numeric_features + categorical_features

        preprocessor = build_preprocessor(numeric_features, categorical_features)
        steps = [("preprocess", preprocessor)]
        if self.polynomial_degree:
            steps.append(("poly", PolynomialFeatures(degree=self.polynomial_degree, include_bias=False)))
        steps.append(("model", self.estimator_factory()))
        self.pipeline = Pipeline(steps)

        X = df_engineered[self.feature_columns]
        y = df_engineered[target_column]
        self.pipeline.fit(X, y)
        return self

    def predict(self, new_df):
        if self.pipeline is None:
            raise RuntimeError("Model has not been fit yet.")
        df_engineered = new_df
        if self.datetime_columns:
            present = [c for c in self.datetime_columns if c in new_df.columns]
            if present:
                df_engineered, _ = engineer_datetime_features(new_df, present)
        missing = [c for c in self.feature_columns if c not in df_engineered.columns]
        if missing:
            raise ValueError(f"Input is missing expected feature columns: {missing}")
        return self.pipeline.predict(df_engineered[self.feature_columns])

    def save(self, path):
        joblib.dump(self, path)

    @staticmethod
    def load(path):
        return joblib.load(path)


def _generic_train_and_predict(data, target_column, estimator_factory, model_key, model_title,

                                feature_columns=None, polynomial_degree=None, test_size=0.2,

                                random_state=42, max_train_samples=None):
    # Reusable training/evaluation engine every model-specific train_and_predict()
    # below delegates to. `data` and `target_column` fully determine behavior --
    # nothing here references any dataset by name.
    model = GenericRegressionModel(estimator_factory, model_key, model_title, feature_columns, polynomial_degree)

    train_df, test_df = train_test_split(data, test_size=test_size, random_state=random_state, shuffle=True)
    if max_train_samples and len(train_df) > max_train_samples:
        train_df = train_df.sample(n=max_train_samples, random_state=random_state)

    start_time = time.time()
    model.fit(train_df, target_column)
    training_time = time.time() - start_time

    predictions = model.predict(test_df)
    y_test = test_df[target_column].values

    return {
        "model": model, "predictions": predictions, "y_test": y_test,
        "training_time": training_time, "test_df": test_df, "train_df": train_df,
    }


# --- Model-specific configuration (everything above this line is fully generic) ---
from sklearn.linear_model import LinearRegression

MODEL_KEY = "linear-regression"
MODEL_TITLE = "Simple Linear Regression"
POLYNOMIAL_DEGREE = None


def _estimator_factory():
    return LinearRegression()


def train_and_predict(data, target_column, feature_columns=None, test_size=0.2, random_state=42, max_train_samples=None):
    # Matches: result = train_and_predict(data=df, target_column="target")
    return _generic_train_and_predict(
        data, target_column, estimator_factory=_estimator_factory, model_key=MODEL_KEY, model_title=MODEL_TITLE,
        feature_columns=feature_columns, polynomial_degree=POLYNOMIAL_DEGREE, test_size=test_size,
        random_state=random_state, max_train_samples=max_train_samples,
    )