based-watermelon commited on
Commit
d34aab0
·
verified ·
1 Parent(s): 59e7b94

Upload lfdmodel.ipynb

Browse files
Files changed (1) hide show
  1. lfdmodel.ipynb +291 -0
lfdmodel.ipynb ADDED
@@ -0,0 +1,291 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "code",
5
+ "execution_count": 20,
6
+ "id": "9800b6a5",
7
+ "metadata": {},
8
+ "outputs": [],
9
+ "source": [
10
+ "import numpy as np\n",
11
+ "import pandas as pd\n",
12
+ "import torch\n",
13
+ "import torch.nn as nn\n",
14
+ "from torch.utils.data import Dataset, DataLoader\n",
15
+ "\n",
16
+ "SEED = 42\n",
17
+ "STATE_DIM = 503\n",
18
+ "N_ACTIONS = 7 \n",
19
+ "\n",
20
+ "\n",
21
+ "N_EPISODES_TRAIN = 8\n",
22
+ "N_EPISODES_VAL = 1\n",
23
+ "\n",
24
+ "torch.manual_seed(SEED)\n",
25
+ "np.random.seed(SEED)\n",
26
+ "\n",
27
+ "\n",
28
+ "class BCDataset(Dataset):\n",
29
+ " #custom dataset\n",
30
+ " def __init__(self, df: pd.DataFrame, episode_ids, state_cols):\n",
31
+ " sub = df[df[\"episode_id\"].isin(episode_ids)]\n",
32
+ " self.X = sub[state_cols].to_numpy(dtype=np.float32).copy()\n",
33
+ " self.y = sub[\"action\"].to_numpy(dtype=np.int64).copy()\n",
34
+ "\n",
35
+ " def __len__(self):\n",
36
+ " return len(self.y)\n",
37
+ "\n",
38
+ " def __getitem__(self, idx):\n",
39
+ " return torch.from_numpy(self.X[idx]), self.y[idx]\n",
40
+ "\n",
41
+ "\n",
42
+ "def split_episodes(episode_ids, n_train=N_EPISODES_TRAIN, n_val=N_EPISODES_VAL, seed=SEED):\n",
43
+ " rng = np.random.default_rng(seed) # random num generator\n",
44
+ " ids = np.array(sorted(episode_ids)) \n",
45
+ " rng.shuffle(ids)\n",
46
+ "\n",
47
+ " n_total = len(ids)\n",
48
+ " if n_train + n_val >= n_total:\n",
49
+ " raise ValueError(\n",
50
+ " f\"Not enough episodes ({n_total}) for N_EPISODES_TRAIN={n_train} \"\n",
51
+ " )\n",
52
+ "\n",
53
+ " train_ids = ids[:n_train]\n",
54
+ " val_ids = ids[n_train:n_train + n_val]\n",
55
+ " test_ids = ids[n_train + n_val:]\n",
56
+ " return train_ids, val_ids, test_ids\n",
57
+ "\n",
58
+ "\n",
59
+ "def load_data(csv_path, batch_size=256, num_workers=0):\n",
60
+ " df = pd.read_csv(csv_path)\n",
61
+ "\n",
62
+ " if \"episode_id\" not in df.columns:\n",
63
+ " raise ValueError(\n",
64
+ " \"Expected an 'episode_id' column\"\n",
65
+ " )\n",
66
+ "\n",
67
+ " state_cols = [f\"s{i}\" for i in range(STATE_DIM)]\n",
68
+ " missing = [c for c in state_cols + [\"action\"] if c not in df.columns]\n",
69
+ " if missing:\n",
70
+ " raise ValueError(f\"CSV is missing expected columns: {missing[:10]}...\")\n",
71
+ "\n",
72
+ " before = len(df)\n",
73
+ " #remove missing columns\n",
74
+ " df = df[pd.to_numeric(df[\"action\"], errors=\"coerce\").isin(range(N_ACTIONS))].copy() \n",
75
+ " df[\"action\"] = df[\"action\"].astype(int)\n",
76
+ " dropped = before - len(df)\n",
77
+ " if dropped:\n",
78
+ " print(f\"Warning: dropped {dropped} rows with invalid action values (corrupted rows)\")\n",
79
+ "\n",
80
+ " episode_ids = df[\"episode_id\"].unique()\n",
81
+ " train_ids, val_ids, test_ids = split_episodes(episode_ids)\n",
82
+ " print(f\"Episodes -> train: {len(train_ids)}, val: {len(val_ids)}, test: {len(test_ids)}\")\n",
83
+ "\n",
84
+ " train_ds = BCDataset(df, train_ids, state_cols)\n",
85
+ " val_ds = BCDataset(df, val_ids, state_cols)\n",
86
+ " test_ds = BCDataset(df, test_ids, state_cols)\n",
87
+ " print(f\"Rows -> train: {len(train_ds)}, val: {len(val_ds)}, test: {len(test_ds)}\")\n",
88
+ "\n",
89
+ " train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=0)\n",
90
+ " val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False, num_workers=0)\n",
91
+ " test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=0)\n",
92
+ "\n",
93
+ " counts = np.bincount(train_ds.y, minlength=N_ACTIONS).astype(np.float32)\n",
94
+ " counts[counts == 0] = 1.0 # preventing div by 0 error provided an action never occurs\n",
95
+ " raw_weights = counts.sum() / (N_ACTIONS * counts) # compensate for rare actions\n",
96
+ " class_weights = np.sqrt(raw_weights)\n",
97
+ " class_weights = np.clip(class_weights, None, 10.0) # cap max weight ratio\n",
98
+ " print(\"Action counts (train):\", counts.astype(int).tolist())\n",
99
+ " print(\"Class weights (sqrt-dampened, capped at 10x):\", np.round(class_weights, 3).tolist())\n",
100
+ "\n",
101
+ " return train_loader, val_loader, test_loader, torch.tensor(class_weights)\n",
102
+ "\n",
103
+ "\n",
104
+ "class BCPolicy(nn.Module):\n",
105
+ "\n",
106
+ " def __init__(self, input_dim=STATE_DIM, hidden=(256, 128), n_actions=N_ACTIONS, dropout=0.2):\n",
107
+ " super().__init__()\n",
108
+ " layers = []\n",
109
+ " prev = input_dim\n",
110
+ " for h in hidden:\n",
111
+ " layers += [nn.Linear(prev, h), nn.ReLU(), nn.Dropout(dropout)]\n",
112
+ " prev = h\n",
113
+ " layers.append(nn.Linear(prev, n_actions))\n",
114
+ " self.net = nn.Sequential(*layers)\n",
115
+ "\n",
116
+ " def forward(self, x):\n",
117
+ " return self.net(x) # logits\n",
118
+ "\n",
119
+ "def run_epoch(model, loader, criterion, optimizer, device, train: bool):\n",
120
+ " if train:\n",
121
+ " model.train()\n",
122
+ " else:\n",
123
+ " model.eval()\n",
124
+ " total_loss, correct, total = 0.0, 0, 0\n",
125
+ "\n",
126
+ " context = torch.enable_grad() if train else torch.no_grad()\n",
127
+ " with context:\n",
128
+ " for X, y in loader:\n",
129
+ " X, y = X.to(device), y.to(device)\n",
130
+ "\n",
131
+ " if train:\n",
132
+ " optimizer.zero_grad()\n",
133
+ "\n",
134
+ " logits = model(X)\n",
135
+ " loss = criterion(logits, y)\n",
136
+ "\n",
137
+ " if train:\n",
138
+ " loss.backward()\n",
139
+ " torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
140
+ " optimizer.step()\n",
141
+ "\n",
142
+ " total_loss += loss.item() * X.size(0)\n",
143
+ " preds = logits.argmax(dim=1)\n",
144
+ " correct += (preds == y).sum().item()\n",
145
+ " total += X.size(0)\n",
146
+ "\n",
147
+ " return total_loss / total, correct / total\n",
148
+ "\n",
149
+ "\n",
150
+ "def train(csv_path, epochs=30, batch_size=256, lr=1e-3, patience=5, out_path=\"bc_policy.pt\"):\n",
151
+ " device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
152
+ " print(\"Device:\", device)\n",
153
+ "\n",
154
+ " train_loader, val_loader, test_loader, class_weights = load_data(csv_path, batch_size)\n",
155
+ "\n",
156
+ " model = BCPolicy().to(device)\n",
157
+ " criterion = nn.CrossEntropyLoss(weight=class_weights.to(device))\n",
158
+ " optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
159
+ "\n",
160
+ " best_val_loss = float(\"inf\")\n",
161
+ " epochs_no_improve = 0\n",
162
+ "\n",
163
+ " for epoch in range(1, epochs + 1):\n",
164
+ " train_loss, train_acc = run_epoch(model, train_loader, criterion, optimizer, device, train=True)\n",
165
+ " val_loss, val_acc = run_epoch(model, val_loader, criterion, optimizer, device, train=False)\n",
166
+ "\n",
167
+ " print(f\"Epoch {epoch:02d} | train loss {train_loss:.4f} acc {train_acc:.3f} \"\n",
168
+ " f\"| val loss {val_loss:.4f} acc {val_acc:.3f}\")\n",
169
+ "\n",
170
+ " if val_loss < best_val_loss - 1e-4:\n",
171
+ " best_val_loss = val_loss\n",
172
+ " epochs_no_improve = 0\n",
173
+ " torch.save(model.state_dict(), out_path)\n",
174
+ " else:\n",
175
+ " epochs_no_improve += 1\n",
176
+ " if epochs_no_improve >= patience:\n",
177
+ " print(f\"Early stopping at epoch {epoch} (no val improvement for {patience} epochs)\")\n",
178
+ " break\n",
179
+ "\n",
180
+ " # evaluate best checkpoint on held-out test episodes\n",
181
+ " model.load_state_dict(torch.load(out_path))\n",
182
+ " test_loss, test_acc = run_epoch(model, test_loader, criterion, optimizer, device, train=False)\n",
183
+ " print(f\"\\nTest (held-out episodes) | loss {test_loss:.4f} acc {test_acc:.3f}\")\n",
184
+ " print(f\"Best model saved to {out_path}\")\n",
185
+ "\n",
186
+ " model.eval()\n",
187
+ " all_preds, all_true = [], []\n",
188
+ " with torch.no_grad():\n",
189
+ " for X, y in test_loader:\n",
190
+ " preds = model(X.to(device)).argmax(dim=1).cpu()\n",
191
+ " all_preds.append(preds)\n",
192
+ " all_true.append(y)\n",
193
+ " all_preds = torch.cat(all_preds).numpy()\n",
194
+ " all_true = torch.cat(all_true).numpy()\n",
195
+ "\n",
196
+ " from sklearn.metrics import classification_report\n",
197
+ " print(classification_report(all_true, all_preds, labels=list(range(N_ACTIONS)), zero_division=0))\n",
198
+ "\n",
199
+ "\n",
200
+ " return model"
201
+ ]
202
+ },
203
+ {
204
+ "cell_type": "code",
205
+ "execution_count": 21,
206
+ "id": "b828e96c",
207
+ "metadata": {},
208
+ "outputs": [
209
+ {
210
+ "name": "stdout",
211
+ "output_type": "stream",
212
+ "text": [
213
+ "Device: cuda\n",
214
+ "Episodes -> train: 8, val: 1, test: 1\n",
215
+ "Rows -> train: 9676, val: 724, test: 1894\n",
216
+ "Action counts (train): [2311, 2550, 60, 69, 3311, 289, 1086]\n",
217
+ "Class weights (sqrt-dampened, capped at 10x): [0.7730000019073486, 0.7360000014305115, 4.800000190734863, 4.47599983215332, 0.6460000276565552, 2.187000036239624, 1.128000020980835]\n",
218
+ "Epoch 01 | train loss 1.4677 acc 0.725 | val loss 0.4536 acc 0.952\n",
219
+ "Epoch 02 | train loss 0.7775 acc 0.834 | val loss 0.2948 acc 0.959\n",
220
+ "Epoch 03 | train loss 0.5922 acc 0.863 | val loss 0.2400 acc 0.959\n",
221
+ "Epoch 04 | train loss 0.5041 acc 0.875 | val loss 0.1976 acc 0.964\n",
222
+ "Epoch 05 | train loss 0.4439 acc 0.885 | val loss 0.2012 acc 0.957\n",
223
+ "Epoch 06 | train loss 0.4016 acc 0.887 | val loss 0.2018 acc 0.943\n",
224
+ "Epoch 07 | train loss 0.3787 acc 0.890 | val loss 0.1931 acc 0.948\n",
225
+ "Epoch 08 | train loss 0.3555 acc 0.890 | val loss 0.1709 acc 0.950\n",
226
+ "Epoch 09 | train loss 0.3414 acc 0.895 | val loss 0.1758 acc 0.950\n",
227
+ "Epoch 10 | train loss 0.3316 acc 0.895 | val loss 0.1639 acc 0.950\n",
228
+ "Epoch 11 | train loss 0.3261 acc 0.899 | val loss 0.1564 acc 0.956\n",
229
+ "Epoch 12 | train loss 0.3199 acc 0.898 | val loss 0.1666 acc 0.950\n",
230
+ "Epoch 13 | train loss 0.3089 acc 0.900 | val loss 0.1549 acc 0.950\n",
231
+ "Epoch 14 | train loss 0.3061 acc 0.901 | val loss 0.1827 acc 0.936\n",
232
+ "Epoch 15 | train loss 0.3061 acc 0.901 | val loss 0.1654 acc 0.941\n",
233
+ "Epoch 16 | train loss 0.2961 acc 0.905 | val loss 0.1783 acc 0.932\n",
234
+ "Epoch 17 | train loss 0.2935 acc 0.904 | val loss 0.1623 acc 0.949\n",
235
+ "Epoch 18 | train loss 0.2889 acc 0.909 | val loss 0.1727 acc 0.941\n",
236
+ "Early stopping at epoch 18 (no val improvement for 5 epochs)\n",
237
+ "\n",
238
+ "Test (held-out episodes) | loss 0.6732 acc 0.825\n",
239
+ "Best model saved to bc_policy.pt\n",
240
+ " precision recall f1-score support\n",
241
+ "\n",
242
+ " 0 0.81 0.88 0.85 488\n",
243
+ " 1 0.84 0.88 0.86 555\n",
244
+ " 2 0.43 0.77 0.56 13\n",
245
+ " 3 0.00 0.00 0.00 0\n",
246
+ " 4 0.93 0.96 0.95 582\n",
247
+ " 5 0.50 0.40 0.44 20\n",
248
+ " 6 0.51 0.28 0.36 236\n",
249
+ "\n",
250
+ " accuracy 0.82 1894\n",
251
+ " macro avg 0.58 0.60 0.57 1894\n",
252
+ "weighted avg 0.81 0.82 0.81 1894\n",
253
+ "\n"
254
+ ]
255
+ }
256
+ ],
257
+ "source": [
258
+ "model = train(\"../dataset_full.csv\", epochs=30, batch_size=256, lr=1e-3, patience=5, out_path=\"bc_policy.pt\")"
259
+ ]
260
+ },
261
+ {
262
+ "cell_type": "code",
263
+ "execution_count": null,
264
+ "id": "8a196a03",
265
+ "metadata": {},
266
+ "outputs": [],
267
+ "source": []
268
+ }
269
+ ],
270
+ "metadata": {
271
+ "kernelspec": {
272
+ "display_name": "Python (torch-env)",
273
+ "language": "python",
274
+ "name": "torch-env"
275
+ },
276
+ "language_info": {
277
+ "codemirror_mode": {
278
+ "name": "ipython",
279
+ "version": 3
280
+ },
281
+ "file_extension": ".py",
282
+ "mimetype": "text/x-python",
283
+ "name": "python",
284
+ "nbconvert_exporter": "python",
285
+ "pygments_lexer": "ipython3",
286
+ "version": "3.11.15"
287
+ }
288
+ },
289
+ "nbformat": 4,
290
+ "nbformat_minor": 5
291
+ }