{ "cells": [ { "cell_type": "code", "execution_count": 20, "id": "9800b6a5", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import pandas as pd\n", "import torch\n", "import torch.nn as nn\n", "from torch.utils.data import Dataset, DataLoader\n", "\n", "SEED = 42\n", "STATE_DIM = 503\n", "N_ACTIONS = 7 \n", "\n", "\n", "N_EPISODES_TRAIN = 8\n", "N_EPISODES_VAL = 1\n", "\n", "torch.manual_seed(SEED)\n", "np.random.seed(SEED)\n", "\n", "\n", "class BCDataset(Dataset):\n", " #custom dataset\n", " def __init__(self, df: pd.DataFrame, episode_ids, state_cols):\n", " sub = df[df[\"episode_id\"].isin(episode_ids)]\n", " self.X = sub[state_cols].to_numpy(dtype=np.float32).copy()\n", " self.y = sub[\"action\"].to_numpy(dtype=np.int64).copy()\n", "\n", " def __len__(self):\n", " return len(self.y)\n", "\n", " def __getitem__(self, idx):\n", " return torch.from_numpy(self.X[idx]), self.y[idx]\n", "\n", "\n", "def split_episodes(episode_ids, n_train=N_EPISODES_TRAIN, n_val=N_EPISODES_VAL, seed=SEED):\n", " rng = np.random.default_rng(seed) # random num generator\n", " ids = np.array(sorted(episode_ids)) \n", " rng.shuffle(ids)\n", "\n", " n_total = len(ids)\n", " if n_train + n_val >= n_total:\n", " raise ValueError(\n", " f\"Not enough episodes ({n_total}) for N_EPISODES_TRAIN={n_train} \"\n", " )\n", "\n", " train_ids = ids[:n_train]\n", " val_ids = ids[n_train:n_train + n_val]\n", " test_ids = ids[n_train + n_val:]\n", " return train_ids, val_ids, test_ids\n", "\n", "\n", "def load_data(csv_path, batch_size=256, num_workers=0):\n", " df = pd.read_csv(csv_path)\n", "\n", " if \"episode_id\" not in df.columns:\n", " raise ValueError(\n", " \"Expected an 'episode_id' column\"\n", " )\n", "\n", " state_cols = [f\"s{i}\" for i in range(STATE_DIM)]\n", " missing = [c for c in state_cols + [\"action\"] if c not in df.columns]\n", " if missing:\n", " raise ValueError(f\"CSV is missing expected columns: {missing[:10]}...\")\n", "\n", " before = len(df)\n", " #remove missing columns\n", " df = df[pd.to_numeric(df[\"action\"], errors=\"coerce\").isin(range(N_ACTIONS))].copy() \n", " df[\"action\"] = df[\"action\"].astype(int)\n", " dropped = before - len(df)\n", " if dropped:\n", " print(f\"Warning: dropped {dropped} rows with invalid action values (corrupted rows)\")\n", "\n", " episode_ids = df[\"episode_id\"].unique()\n", " train_ids, val_ids, test_ids = split_episodes(episode_ids)\n", " print(f\"Episodes -> train: {len(train_ids)}, val: {len(val_ids)}, test: {len(test_ids)}\")\n", "\n", " train_ds = BCDataset(df, train_ids, state_cols)\n", " val_ds = BCDataset(df, val_ids, state_cols)\n", " test_ds = BCDataset(df, test_ids, state_cols)\n", " print(f\"Rows -> train: {len(train_ds)}, val: {len(val_ds)}, test: {len(test_ds)}\")\n", "\n", " train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=0)\n", " val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False, num_workers=0)\n", " test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=0)\n", "\n", " counts = np.bincount(train_ds.y, minlength=N_ACTIONS).astype(np.float32)\n", " counts[counts == 0] = 1.0 # preventing div by 0 error provided an action never occurs\n", " raw_weights = counts.sum() / (N_ACTIONS * counts) # compensate for rare actions\n", " class_weights = np.sqrt(raw_weights)\n", " class_weights = np.clip(class_weights, None, 10.0) # cap max weight ratio\n", " print(\"Action counts (train):\", counts.astype(int).tolist())\n", " print(\"Class weights (sqrt-dampened, capped at 10x):\", np.round(class_weights, 3).tolist())\n", "\n", " return train_loader, val_loader, test_loader, torch.tensor(class_weights)\n", "\n", "\n", "class BCPolicy(nn.Module):\n", "\n", " def __init__(self, input_dim=STATE_DIM, hidden=(256, 128), n_actions=N_ACTIONS, dropout=0.2):\n", " super().__init__()\n", " layers = []\n", " prev = input_dim\n", " for h in hidden:\n", " layers += [nn.Linear(prev, h), nn.ReLU(), nn.Dropout(dropout)]\n", " prev = h\n", " layers.append(nn.Linear(prev, n_actions))\n", " self.net = nn.Sequential(*layers)\n", "\n", " def forward(self, x):\n", " return self.net(x) # logits\n", "\n", "def run_epoch(model, loader, criterion, optimizer, device, train: bool):\n", " if train:\n", " model.train()\n", " else:\n", " model.eval()\n", " total_loss, correct, total = 0.0, 0, 0\n", "\n", " context = torch.enable_grad() if train else torch.no_grad()\n", " with context:\n", " for X, y in loader:\n", " X, y = X.to(device), y.to(device)\n", "\n", " if train:\n", " optimizer.zero_grad()\n", "\n", " logits = model(X)\n", " loss = criterion(logits, y)\n", "\n", " if train:\n", " loss.backward()\n", " torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n", " optimizer.step()\n", "\n", " total_loss += loss.item() * X.size(0)\n", " preds = logits.argmax(dim=1)\n", " correct += (preds == y).sum().item()\n", " total += X.size(0)\n", "\n", " return total_loss / total, correct / total\n", "\n", "\n", "def train(csv_path, epochs=30, batch_size=256, lr=1e-3, patience=5, out_path=\"bc_policy.pt\"):\n", " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", " print(\"Device:\", device)\n", "\n", " train_loader, val_loader, test_loader, class_weights = load_data(csv_path, batch_size)\n", "\n", " model = BCPolicy().to(device)\n", " criterion = nn.CrossEntropyLoss(weight=class_weights.to(device))\n", " optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n", "\n", " best_val_loss = float(\"inf\")\n", " epochs_no_improve = 0\n", "\n", " for epoch in range(1, epochs + 1):\n", " train_loss, train_acc = run_epoch(model, train_loader, criterion, optimizer, device, train=True)\n", " val_loss, val_acc = run_epoch(model, val_loader, criterion, optimizer, device, train=False)\n", "\n", " print(f\"Epoch {epoch:02d} | train loss {train_loss:.4f} acc {train_acc:.3f} \"\n", " f\"| val loss {val_loss:.4f} acc {val_acc:.3f}\")\n", "\n", " if val_loss < best_val_loss - 1e-4:\n", " best_val_loss = val_loss\n", " epochs_no_improve = 0\n", " torch.save(model.state_dict(), out_path)\n", " else:\n", " epochs_no_improve += 1\n", " if epochs_no_improve >= patience:\n", " print(f\"Early stopping at epoch {epoch} (no val improvement for {patience} epochs)\")\n", " break\n", "\n", " # evaluate best checkpoint on held-out test episodes\n", " model.load_state_dict(torch.load(out_path))\n", " test_loss, test_acc = run_epoch(model, test_loader, criterion, optimizer, device, train=False)\n", " print(f\"\\nTest (held-out episodes) | loss {test_loss:.4f} acc {test_acc:.3f}\")\n", " print(f\"Best model saved to {out_path}\")\n", "\n", " model.eval()\n", " all_preds, all_true = [], []\n", " with torch.no_grad():\n", " for X, y in test_loader:\n", " preds = model(X.to(device)).argmax(dim=1).cpu()\n", " all_preds.append(preds)\n", " all_true.append(y)\n", " all_preds = torch.cat(all_preds).numpy()\n", " all_true = torch.cat(all_true).numpy()\n", "\n", " from sklearn.metrics import classification_report\n", " print(classification_report(all_true, all_preds, labels=list(range(N_ACTIONS)), zero_division=0))\n", "\n", "\n", " return model" ] }, { "cell_type": "code", "execution_count": 21, "id": "b828e96c", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Device: cuda\n", "Episodes -> train: 8, val: 1, test: 1\n", "Rows -> train: 9676, val: 724, test: 1894\n", "Action counts (train): [2311, 2550, 60, 69, 3311, 289, 1086]\n", "Class weights (sqrt-dampened, capped at 10x): [0.7730000019073486, 0.7360000014305115, 4.800000190734863, 4.47599983215332, 0.6460000276565552, 2.187000036239624, 1.128000020980835]\n", "Epoch 01 | train loss 1.4677 acc 0.725 | val loss 0.4536 acc 0.952\n", "Epoch 02 | train loss 0.7775 acc 0.834 | val loss 0.2948 acc 0.959\n", "Epoch 03 | train loss 0.5922 acc 0.863 | val loss 0.2400 acc 0.959\n", "Epoch 04 | train loss 0.5041 acc 0.875 | val loss 0.1976 acc 0.964\n", "Epoch 05 | train loss 0.4439 acc 0.885 | val loss 0.2012 acc 0.957\n", "Epoch 06 | train loss 0.4016 acc 0.887 | val loss 0.2018 acc 0.943\n", "Epoch 07 | train loss 0.3787 acc 0.890 | val loss 0.1931 acc 0.948\n", "Epoch 08 | train loss 0.3555 acc 0.890 | val loss 0.1709 acc 0.950\n", "Epoch 09 | train loss 0.3414 acc 0.895 | val loss 0.1758 acc 0.950\n", "Epoch 10 | train loss 0.3316 acc 0.895 | val loss 0.1639 acc 0.950\n", "Epoch 11 | train loss 0.3261 acc 0.899 | val loss 0.1564 acc 0.956\n", "Epoch 12 | train loss 0.3199 acc 0.898 | val loss 0.1666 acc 0.950\n", "Epoch 13 | train loss 0.3089 acc 0.900 | val loss 0.1549 acc 0.950\n", "Epoch 14 | train loss 0.3061 acc 0.901 | val loss 0.1827 acc 0.936\n", "Epoch 15 | train loss 0.3061 acc 0.901 | val loss 0.1654 acc 0.941\n", "Epoch 16 | train loss 0.2961 acc 0.905 | val loss 0.1783 acc 0.932\n", "Epoch 17 | train loss 0.2935 acc 0.904 | val loss 0.1623 acc 0.949\n", "Epoch 18 | train loss 0.2889 acc 0.909 | val loss 0.1727 acc 0.941\n", "Early stopping at epoch 18 (no val improvement for 5 epochs)\n", "\n", "Test (held-out episodes) | loss 0.6732 acc 0.825\n", "Best model saved to bc_policy.pt\n", " precision recall f1-score support\n", "\n", " 0 0.81 0.88 0.85 488\n", " 1 0.84 0.88 0.86 555\n", " 2 0.43 0.77 0.56 13\n", " 3 0.00 0.00 0.00 0\n", " 4 0.93 0.96 0.95 582\n", " 5 0.50 0.40 0.44 20\n", " 6 0.51 0.28 0.36 236\n", "\n", " accuracy 0.82 1894\n", " macro avg 0.58 0.60 0.57 1894\n", "weighted avg 0.81 0.82 0.81 1894\n", "\n" ] } ], "source": [ "model = train(\"../dataset_full.csv\", epochs=30, batch_size=256, lr=1e-3, patience=5, out_path=\"bc_policy.pt\")" ] }, { "cell_type": "code", "execution_count": null, "id": "8a196a03", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python (torch-env)", "language": "python", "name": "torch-env" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.15" } }, "nbformat": 4, "nbformat_minor": 5 }