from __future__ import annotations from dataclasses import dataclass from pathlib import Path import html import json import random import sys import numpy as np import pandas as pd import torch from torch import nn from torch.utils.data import DataLoader, TensorDataset SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_ROOT = SCRIPT_DIR.parent MODEL_DIR = PROJECT_ROOT / "data" / "modeling" REPORTS_DIR = PROJECT_ROOT / "reports" REPORT_PATH = REPORTS_DIR / "attack_prediction_v2_experiments.html" JSON_PATH = MODEL_DIR / "attack_prediction_v2_experiments.json" BEST_MODEL_PATH = MODEL_DIR / "attack_prediction_v2_best_bundle.pt" BEST_PREDICTIONS_PATH = MODEL_DIR / "attack_prediction_v2_best_test_predictions.parquet" if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) import train_attack_prediction_ffn as base # noqa: E402 RANDOM_SEED = 42 BATCH_SIZE = 256 MAX_EPOCHS = 220 PATIENCE = 28 @dataclass class SplitData: x: np.ndarray y_attack: np.ndarray y_pv: np.ndarray baseline_attack: np.ndarray baseline_pv_scaled: np.ndarray metadata: pd.DataFrame class BaseModel(nn.Module): def forward( self, x: torch.Tensor, baseline_attack: torch.Tensor, baseline_pv_scaled: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: raise NotImplementedError class FFNModel(BaseModel): def __init__(self, input_dim: int, attack_dim: int, pv_dim: int, hidden_dims: list[int], dropouts: list[float]) -> None: super().__init__() layers: list[nn.Module] = [] prev = input_dim for width, drop in zip(hidden_dims, dropouts): layers.extend([nn.Linear(prev, width), nn.ReLU(), nn.Dropout(drop)]) prev = width self.backbone = nn.Sequential(*layers) self.attack_head = nn.Linear(prev, attack_dim) self.pv_head = nn.Linear(prev, pv_dim) def forward( self, x: torch.Tensor, baseline_attack: torch.Tensor, baseline_pv_scaled: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: h = self.backbone(x) attack = torch.softmax(self.attack_head(h), dim=1) pv = self.pv_head(h) return attack, pv class ResidualModel(BaseModel): def __init__(self, input_dim: int, attack_dim: int, pv_dim: int, hidden_dims: list[int], dropouts: list[float]) -> None: super().__init__() layers: list[nn.Module] = [] prev = input_dim for width, drop in zip(hidden_dims, dropouts): layers.extend([nn.Linear(prev, width), nn.ReLU(), nn.Dropout(drop)]) prev = width self.backbone = nn.Sequential(*layers) self.attack_delta = nn.Linear(prev, attack_dim) self.pv_delta = nn.Linear(prev, pv_dim) def forward( self, x: torch.Tensor, baseline_attack: torch.Tensor, baseline_pv_scaled: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: h = self.backbone(x) attack_logits = torch.log(torch.clamp(baseline_attack, min=1e-6)) + self.attack_delta(h) attack = torch.softmax(attack_logits, dim=1) pv = baseline_pv_scaled + self.pv_delta(h) return attack, pv def _set_seed(seed: int = RANDOM_SEED) -> None: random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) def _make_loader(split: SplitData, shuffle: bool) -> DataLoader: dataset = TensorDataset( torch.from_numpy(split.x), torch.from_numpy(split.y_attack), torch.from_numpy(split.y_pv), torch.from_numpy(split.baseline_attack), torch.from_numpy(split.baseline_pv_scaled), ) return DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=shuffle) def _loss_fn(pred_attack: torch.Tensor, pred_pv: torch.Tensor, y_attack: torch.Tensor, y_pv: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: attack_loss = torch.mean((pred_attack - y_attack) ** 2) pv_loss = torch.nn.functional.smooth_l1_loss(pred_pv, y_pv) return attack_loss + pv_loss, attack_loss, pv_loss def _evaluate_loader(model: BaseModel, loader: DataLoader, device: torch.device) -> dict[str, float]: model.eval() total_loss = 0.0 attack_loss = 0.0 pv_loss = 0.0 n_batches = 0 with torch.no_grad(): for x, y_attack, y_pv, baseline_attack, baseline_pv_scaled in loader: x = x.to(device) y_attack = y_attack.to(device) y_pv = y_pv.to(device) baseline_attack = baseline_attack.to(device) baseline_pv_scaled = baseline_pv_scaled.to(device) pred_attack, pred_pv = model(x, baseline_attack, baseline_pv_scaled) loss, la, lp = _loss_fn(pred_attack, pred_pv, y_attack, y_pv) total_loss += float(loss.item()) attack_loss += float(la.item()) pv_loss += float(lp.item()) n_batches += 1 return { "loss": total_loss / max(n_batches, 1), "attack_loss": attack_loss / max(n_batches, 1), "pv_loss": pv_loss / max(n_batches, 1), } def _train_model(model: BaseModel, train: SplitData, val: SplitData, lr: float, weight_decay: float) -> tuple[BaseModel, list[dict[str, float]]]: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = model.to(device) optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) train_loader = _make_loader(train, shuffle=True) val_loader = _make_loader(val, shuffle=False) best_state = None best_val = float("inf") history: list[dict[str, float]] = [] patience_left = PATIENCE for epoch in range(1, MAX_EPOCHS + 1): model.train() running_total = 0.0 n_batches = 0 for x, y_attack, y_pv, baseline_attack, baseline_pv_scaled in train_loader: x = x.to(device) y_attack = y_attack.to(device) y_pv = y_pv.to(device) baseline_attack = baseline_attack.to(device) baseline_pv_scaled = baseline_pv_scaled.to(device) optimizer.zero_grad() pred_attack, pred_pv = model(x, baseline_attack, baseline_pv_scaled) loss, _, _ = _loss_fn(pred_attack, pred_pv, y_attack, y_pv) loss.backward() optimizer.step() running_total += float(loss.item()) n_batches += 1 val_metrics = _evaluate_loader(model, val_loader, device) history.append( { "epoch": epoch, "train_loss": running_total / max(n_batches, 1), "val_loss": val_metrics["loss"], "val_attack_loss": val_metrics["attack_loss"], "val_pv_loss": val_metrics["pv_loss"], } ) if val_metrics["loss"] < best_val - 1e-6: best_val = val_metrics["loss"] best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} patience_left = PATIENCE else: patience_left -= 1 if patience_left <= 0: break if best_state is None: best_state = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()} model.load_state_dict(best_state) return model, history def _predict(model: BaseModel, split: SplitData) -> tuple[np.ndarray, np.ndarray]: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.eval() attack_preds: list[np.ndarray] = [] pv_preds: list[np.ndarray] = [] loader = _make_loader(split, shuffle=False) with torch.no_grad(): for x, _, _, baseline_attack, baseline_pv_scaled in loader: x = x.to(device) baseline_attack = baseline_attack.to(device) baseline_pv_scaled = baseline_pv_scaled.to(device) attack, pv = model(x, baseline_attack, baseline_pv_scaled) attack_preds.append(attack.cpu().numpy()) pv_preds.append(pv.cpu().numpy()) return np.vstack(attack_preds), np.vstack(pv_preds) def _prepare_data() -> tuple[pd.DataFrame, pd.DataFrame, list[str], list[str], np.ndarray, np.ndarray, dict[str, np.ndarray], np.ndarray, np.ndarray, np.ndarray]: df = base._load_dataset() df = df[df["usable_for_model"]].copy().reset_index(drop=True) features, _, attack_targets, pv_targets, _ = base._feature_matrix(df) train_df = df[df["split"] == "train"].copy() train_idx_pd, val_idx_pd, _ = base._build_temporal_validation(train_df) x_scaled, _ = base._standardize_features(features, train_idx_pd) y_attack = df[attack_targets].to_numpy(dtype=np.float32) y_pv_raw = df[pv_targets].to_numpy(dtype=np.float32) pv_train_raw = y_pv_raw[train_idx_pd.to_numpy()] pv_scaler = { "mean": pv_train_raw.mean(axis=0, keepdims=True).astype(np.float32), "std": np.where(pv_train_raw.std(axis=0, keepdims=True) > 0, pv_train_raw.std(axis=0, keepdims=True), 1.0).astype(np.float32), } y_pv = ((y_pv_raw - pv_scaler["mean"]) / pv_scaler["std"]).astype(np.float32) baseline_attack_cols = [base._baseline_col_from_target(c) for c in attack_targets] baseline_pv_cols = [base._baseline_col_from_target(c) for c in pv_targets] baseline_attack = base._normalize_rows(df[baseline_attack_cols].to_numpy(dtype=float)).astype(np.float32) baseline_pv_scaled = ((df[baseline_pv_cols].to_numpy(dtype=np.float32) - pv_scaler["mean"]) / pv_scaler["std"]).astype(np.float32) return ( df, x_scaled, attack_targets, pv_targets, y_attack, y_pv, pv_scaler, baseline_attack, baseline_pv_scaled, train_idx_pd.to_numpy(), val_idx_pd.to_numpy(), df.index[df["split"] == "test"].to_numpy(), ) def _pack_split( df: pd.DataFrame, x: np.ndarray, y_attack: np.ndarray, y_pv: np.ndarray, baseline_attack: np.ndarray, baseline_pv_scaled: np.ndarray, idx: np.ndarray, ) -> SplitData: return SplitData( x=x[idx].astype(np.float32), y_attack=y_attack[idx].astype(np.float32), y_pv=y_pv[idx].astype(np.float32), baseline_attack=baseline_attack[idx].astype(np.float32), baseline_pv_scaled=baseline_pv_scaled[idx].astype(np.float32), metadata=df.iloc[idx].copy().reset_index(drop=True), ) def _evaluate_predictions( metadata: pd.DataFrame, attack_targets: list[str], pv_targets: list[str], attack_pred: np.ndarray, pv_pred: np.ndarray, ) -> dict[str, float]: baseline_attack_cols = [base._baseline_col_from_target(c) for c in attack_targets] baseline_pv_cols = [base._baseline_col_from_target(c) for c in pv_targets] true_attack = metadata[attack_targets].to_numpy(dtype=float) true_pv = metadata[pv_targets].to_numpy(dtype=float) baseline_attack = base._normalize_rows(metadata[baseline_attack_cols].to_numpy(dtype=float)) baseline_pv = metadata[baseline_pv_cols].to_numpy(dtype=float) return { "attack_mae_model": base._mean_abs_error(true_attack, attack_pred), "attack_mae_baseline": base._mean_abs_error(true_attack, baseline_attack), "attack_jsd_model": base._jsd_mean(true_attack, attack_pred), "attack_jsd_baseline": base._jsd_mean(true_attack, baseline_attack), "pv_mae_model": base._mean_abs_error(true_pv, pv_pred), "pv_mae_baseline": base._mean_abs_error(true_pv, baseline_pv), "pv_r2_model": base._r2_score_mean(true_pv, pv_pred), "pv_r2_baseline": base._r2_score_mean(true_pv, baseline_pv), } def _run_experiment( df: pd.DataFrame, x_scaled_df: pd.DataFrame, attack_targets: list[str], pv_targets: list[str], y_attack: np.ndarray, y_pv: np.ndarray, pv_scaler: dict[str, np.ndarray], baseline_attack: np.ndarray, baseline_pv_scaled: np.ndarray, train_idx: np.ndarray, val_idx: np.ndarray, test_idx: np.ndarray, name: str, hidden_dims: list[int], dropouts: list[float], lr: float, weight_decay: float, residual: bool, ) -> dict: x = x_scaled_df.to_numpy(dtype=np.float32) train_split = _pack_split(df, x, y_attack, y_pv, baseline_attack, baseline_pv_scaled, train_idx) val_split = _pack_split(df, x, y_attack, y_pv, baseline_attack, baseline_pv_scaled, val_idx) test_split = _pack_split(df, x, y_attack, y_pv, baseline_attack, baseline_pv_scaled, test_idx) if residual: model = ResidualModel(len(x_scaled_df.columns), len(attack_targets), len(pv_targets), hidden_dims, dropouts) else: model = FFNModel(len(x_scaled_df.columns), len(attack_targets), len(pv_targets), hidden_dims, dropouts) model, history = _train_model(model, train_split, val_split, lr=lr, weight_decay=weight_decay) val_attack_pred, val_pv_scaled = _predict(model, val_split) test_attack_pred, test_pv_scaled = _predict(model, test_split) val_pv_pred = (val_pv_scaled * pv_scaler["std"]) + pv_scaler["mean"] test_pv_pred = (test_pv_scaled * pv_scaler["std"]) + pv_scaler["mean"] return { "name": name, "residual": residual, "hidden_dims": hidden_dims, "dropouts": dropouts, "lr": lr, "weight_decay": weight_decay, "history": history, "val_metrics": _evaluate_predictions(val_split.metadata, attack_targets, pv_targets, val_attack_pred, val_pv_pred), "test_metrics": _evaluate_predictions(test_split.metadata, attack_targets, pv_targets, test_attack_pred, test_pv_pred), "model_state_dict": {k: v.detach().cpu() for k, v in model.state_dict().items()}, "test_attack_pred": test_attack_pred, "test_pv_pred": test_pv_pred, } def _results_table(results: list[dict], split_key: str) -> pd.DataFrame: rows = [] for r in results: m = r[f"{split_key}_metrics"] rows.append( { "modelo": r["name"], "residual": r["residual"], "attack_mae_model": m["attack_mae_model"], "attack_mae_baseline": m["attack_mae_baseline"], "attack_delta": m["attack_mae_baseline"] - m["attack_mae_model"], "attack_jsd_model": m["attack_jsd_model"], "attack_jsd_baseline": m["attack_jsd_baseline"], "pv_mae_model": m["pv_mae_model"], "pv_mae_baseline": m["pv_mae_baseline"], "pv_delta": m["pv_mae_baseline"] - m["pv_mae_model"], "pv_r2_model": m["pv_r2_model"], "pv_r2_baseline": m["pv_r2_baseline"], } ) return pd.DataFrame(rows).sort_values(["attack_mae_model", "pv_mae_model"]).reset_index(drop=True) def _render_table(title: str, df: pd.DataFrame) -> str: rows = [] for _, row in df.iterrows(): rows.append( "
| Modelo | Residual | Attack MAE | Baseline | Mejora | Attack JSD | Baseline | PV MAE | Baseline | Mejora | PV R2 | Baseline R2 |
|---|
| Fecha | Rival | Attack MAE modelo | Attack MAE baseline | PV MAE modelo | PV MAE baseline |
|---|
Segunda tanda enfocada en mejorar ataque: nuevas configuraciones, una red bastante mas grande y un modelo residual que parte del baseline de temporada y aprende una correccion.
{html.escape(summary['best_model'])}
{summary['train_rows']}
{summary['val_rows']}
{summary['test_rows']}