Spaces:
Sleeping
Sleeping
| import numpy as np | |
| import pandas as pd | |
| def _make_data(rng: np.random.Generator, n: int) -> tuple[pd.DataFrame, np.ndarray]: | |
| """Generate a dataset with moderate separability so that proper training yields ~0.72-0.78 accuracy. | |
| Deterministic given rng. | |
| Returns (df, y). | |
| """ | |
| # Binary labels 0/1 | |
| y = rng.integers(0, 2, size=n) | |
| # Features: class-dependent means with substantial noise | |
| # This yields a problem that is learnable but not near-perfect. | |
| shift = (y * 2 - 1).reshape(-1, 1) # -1 for class 0, +1 for class 1 | |
| base = rng.normal(0, 1.5, size=(n, 5)) | |
| means = np.array([0.8, 0.6, 0.5, 0.3, 0.0]) | |
| X = base + shift * means # class-separable signal | |
| # Add one weakly informative interaction-like feature | |
| x_extra = (X[:, 0] * 0.3 + X[:, 1] * -0.2 + rng.normal(0, 0.7, size=n)).reshape(-1, 1) | |
| X = np.concatenate([X, x_extra], axis=1) | |
| df = pd.DataFrame(X, columns=[ | |
| "feat_1", | |
| "feat_2", | |
| "feat_3", | |
| "feat_4", | |
| "feat_5", | |
| "feat_6", | |
| ]) | |
| df["target"] = y | |
| return df, y | |
| def generate_scenario() -> dict: | |
| """ | |
| Task 3 (Hard): Data leakage via scaling before split. | |
| - Data: ~1000 rows; model accuracy appears suspiciously high when StandardScaler is fit on the full dataset | |
| - Bug: StandardScaler.fit_transform applied before train_test_split (data leakage) | |
| - True accuracy after fixing scaling (fit on train only) should drop to ~0.72-0.78 | |
| - Also returns held_out_X, held_out_y (NOT to be exposed externally) | |
| - Deterministic via seed=42 | |
| """ | |
| rng = np.random.default_rng(42) | |
| # Main dataset for the episode | |
| df, y = _make_data(rng, 1000) | |
| # Held-out set for grader3 only | |
| held_out_df, held_out_y = _make_data(rng, 200) | |
| held_out_X = held_out_df.drop(columns=["target"]).to_numpy() | |
| broken_script = ( | |
| "# Broken: fits StandardScaler on FULL dataset before splitting (data leakage)\n" | |
| "import numpy as np\n" | |
| "from sklearn.preprocessing import StandardScaler\n" | |
| "from sklearn.linear_model import LogisticRegression\n" | |
| "from sklearn.model_selection import train_test_split\n" | |
| "\n" | |
| "X = df.drop(columns=['target']).to_numpy()\n" | |
| "y = df['target'].to_numpy()\n" | |
| "scaler = StandardScaler()\n" | |
| "X_scaled = scaler.fit_transform(X) # data leakage: fit on full data\n" | |
| "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n" | |
| "clf = LogisticRegression(max_iter=1000, random_state=42)\n" | |
| "clf.fit(X_train, y_train)\n" | |
| "acc = clf.score(X_test, y_test)\n" | |
| "print('Accuracy:', round(acc, 4))\n" | |
| ) | |
| # No error on run; it will print an accuracy that looks inflated. | |
| initial_error_log = "" | |
| task_description = ( | |
| "The script performs StandardScaler.fit_transform on the entire dataset before splitting into " | |
| "train/test, leaking information. This yields a suspiciously high reported accuracy." | |
| ) | |
| expected_fix_hint = ( | |
| "Move scaler.fit to occur AFTER train_test_split and fit ONLY on X_train, then transform X_train and X_test." | |
| ) | |
| return { | |
| "broken_script": broken_script, | |
| "dataframe": df, | |
| "initial_error_log": initial_error_log, | |
| "task_description": task_description, | |
| "expected_fix_hint": expected_fix_hint, | |
| # Hidden extras for grader3 (never exposed via API) | |
| "held_out_X": held_out_X, | |
| "held_out_y": held_out_y, | |
| "full_df": df, | |
| } | |