Sync run_colab_remaining_ablations.py with 6 complete empirical runs and Tathagat Sir quotes
Browse files- run_colab_remaining_ablations.py +64 -11
run_colab_remaining_ablations.py
CHANGED
|
@@ -245,11 +245,12 @@ class TransformerAttentionBlock(nn.Module):
|
|
| 245 |
return self.norm2(x + ffn_out)
|
| 246 |
|
| 247 |
class BioHCLSSSM_ExperimentModel(nn.Module):
|
| 248 |
-
def __init__(self, sequence_engine="sssm", num_blocks=3, embed_dim=512, d_state=64):
|
| 249 |
super().__init__()
|
| 250 |
self.hier_cnn = HierarchicalConvBlock(3, embed_dim // 2)
|
| 251 |
self.backbone = BackboneExtractor(embed_dim // 2)
|
| 252 |
self.fusion = nn.Linear(embed_dim, embed_dim)
|
|
|
|
| 253 |
|
| 254 |
self.blocks = nn.ModuleList()
|
| 255 |
for i in range(num_blocks):
|
|
@@ -268,43 +269,70 @@ class BioHCLSSSM_ExperimentModel(nn.Module):
|
|
| 268 |
|
| 269 |
h = tokens
|
| 270 |
for block in self.blocks:
|
| 271 |
-
|
|
|
|
| 272 |
|
| 273 |
feat = self.norm(h).mean(dim=1)
|
| 274 |
logits = self.head(feat)
|
| 275 |
return logits, feat
|
| 276 |
|
| 277 |
# -------------------------------------------------------------
|
| 278 |
-
# 4. EXECUTION MATRIX (
|
| 279 |
# -------------------------------------------------------------
|
| 280 |
runs_to_execute = [
|
| 281 |
{
|
| 282 |
"run_id": "Table3_Run1_lambda_0.2",
|
| 283 |
-
"category": "
|
| 284 |
"sequence_engine": "sssm",
|
|
|
|
|
|
|
| 285 |
"lambda_bio": 0.2,
|
| 286 |
"description": "Bio-PINN loss weighting with light anatomical constraint (lambda=0.2)"
|
| 287 |
},
|
| 288 |
{
|
| 289 |
"run_id": "Table3_Run2_lambda_0.5",
|
| 290 |
-
"category": "
|
| 291 |
"sequence_engine": "sssm",
|
|
|
|
|
|
|
| 292 |
"lambda_bio": 0.5,
|
| 293 |
"description": "Bio-PINN loss weighting with moderate anatomical constraint (lambda=0.5)"
|
| 294 |
},
|
| 295 |
{
|
| 296 |
"run_id": "Table3_Run3_lambda_1.0",
|
| 297 |
-
"category": "
|
| 298 |
"sequence_engine": "sssm",
|
|
|
|
|
|
|
| 299 |
"lambda_bio": 1.0,
|
| 300 |
-
"description": "Bio-PINN loss weighting with over-weighted constraint (lambda=1.0)"
|
| 301 |
},
|
| 302 |
{
|
| 303 |
"run_id": "Table1_Run4_MHSA_Attention",
|
| 304 |
-
"category": "
|
| 305 |
"sequence_engine": "attention",
|
|
|
|
|
|
|
| 306 |
"lambda_bio": 0.7,
|
| 307 |
-
"description": "SSSM replaced with Multi-Head Self-Attention (MHSA, 8 heads)"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 308 |
}
|
| 309 |
]
|
| 310 |
|
|
@@ -323,13 +351,34 @@ for run_idx, run_config in enumerate(runs_to_execute):
|
|
| 323 |
print(f"\n>>> Running [{run_idx + 1}/{len(runs_to_execute)}]: {run_config['run_id']}")
|
| 324 |
print(f" Description: {run_config['description']}")
|
| 325 |
|
|
|
|
|
|
|
| 326 |
model = BioHCLSSSM_ExperimentModel(
|
| 327 |
sequence_engine=run_config['sequence_engine'],
|
| 328 |
-
num_blocks=
|
| 329 |
embed_dim=512,
|
| 330 |
-
d_state=64
|
|
|
|
| 331 |
).to(DEVICE)
|
| 332 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 333 |
optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-4)
|
| 334 |
scaler = torch.amp.GradScaler('cuda') if DEVICE.type == 'cuda' else None
|
| 335 |
|
|
@@ -415,7 +464,11 @@ for run_idx, run_config in enumerate(runs_to_execute):
|
|
| 415 |
"run_id": run_config['run_id'],
|
| 416 |
"category": run_config['category'],
|
| 417 |
"sequence_engine": run_config['sequence_engine'],
|
|
|
|
|
|
|
| 418 |
"lambda_bio": run_config['lambda_bio'],
|
|
|
|
|
|
|
| 419 |
"val_acc": best_metrics.get("val_acc", 0.0),
|
| 420 |
"val_f1": best_metrics.get("val_f1", 0.0),
|
| 421 |
"test_acc_pct": round(t_acc * 100, 2),
|
|
|
|
| 245 |
return self.norm2(x + ffn_out)
|
| 246 |
|
| 247 |
class BioHCLSSSM_ExperimentModel(nn.Module):
|
| 248 |
+
def __init__(self, sequence_engine="sssm", num_blocks=3, embed_dim=512, d_state=64, use_skip=True):
|
| 249 |
super().__init__()
|
| 250 |
self.hier_cnn = HierarchicalConvBlock(3, embed_dim // 2)
|
| 251 |
self.backbone = BackboneExtractor(embed_dim // 2)
|
| 252 |
self.fusion = nn.Linear(embed_dim, embed_dim)
|
| 253 |
+
self.use_skip = use_skip
|
| 254 |
|
| 255 |
self.blocks = nn.ModuleList()
|
| 256 |
for i in range(num_blocks):
|
|
|
|
| 269 |
|
| 270 |
h = tokens
|
| 271 |
for block in self.blocks:
|
| 272 |
+
res = h if self.use_skip else None
|
| 273 |
+
h = block(h, res)
|
| 274 |
|
| 275 |
feat = self.norm(h).mean(dim=1)
|
| 276 |
logits = self.head(feat)
|
| 277 |
return logits, feat
|
| 278 |
|
| 279 |
# -------------------------------------------------------------
|
| 280 |
+
# 4. EXECUTION MATRIX (6 COMPLETE RUNS FOR TABLES 1, 2, & 3)
|
| 281 |
# -------------------------------------------------------------
|
| 282 |
runs_to_execute = [
|
| 283 |
{
|
| 284 |
"run_id": "Table3_Run1_lambda_0.2",
|
| 285 |
+
"category": "Table3_Loss_Weighting",
|
| 286 |
"sequence_engine": "sssm",
|
| 287 |
+
"num_blocks": 3,
|
| 288 |
+
"use_skip": True,
|
| 289 |
"lambda_bio": 0.2,
|
| 290 |
"description": "Bio-PINN loss weighting with light anatomical constraint (lambda=0.2)"
|
| 291 |
},
|
| 292 |
{
|
| 293 |
"run_id": "Table3_Run2_lambda_0.5",
|
| 294 |
+
"category": "Table3_Loss_Weighting",
|
| 295 |
"sequence_engine": "sssm",
|
| 296 |
+
"num_blocks": 3,
|
| 297 |
+
"use_skip": True,
|
| 298 |
"lambda_bio": 0.5,
|
| 299 |
"description": "Bio-PINN loss weighting with moderate anatomical constraint (lambda=0.5)"
|
| 300 |
},
|
| 301 |
{
|
| 302 |
"run_id": "Table3_Run3_lambda_1.0",
|
| 303 |
+
"category": "Table3_Loss_Weighting",
|
| 304 |
"sequence_engine": "sssm",
|
| 305 |
+
"num_blocks": 3,
|
| 306 |
+
"use_skip": True,
|
| 307 |
"lambda_bio": 1.0,
|
| 308 |
+
"description": "Bio-PINN loss weighting with over-weighted constraint (lambda=1.0, over-regularization penalty test)"
|
| 309 |
},
|
| 310 |
{
|
| 311 |
"run_id": "Table1_Run4_MHSA_Attention",
|
| 312 |
+
"category": "Table1_Component_Ablation",
|
| 313 |
"sequence_engine": "attention",
|
| 314 |
+
"num_blocks": 3,
|
| 315 |
+
"use_skip": True,
|
| 316 |
"lambda_bio": 0.7,
|
| 317 |
+
"description": "SSSM replaced with Multi-Head Self-Attention (MHSA, 8 heads, quadratic complexity benchmark)"
|
| 318 |
+
},
|
| 319 |
+
{
|
| 320 |
+
"run_id": "Table2_Run5_Depth_L1",
|
| 321 |
+
"category": "Table2_Layer_Deduction",
|
| 322 |
+
"sequence_engine": "sssm",
|
| 323 |
+
"num_blocks": 1,
|
| 324 |
+
"use_skip": True,
|
| 325 |
+
"lambda_bio": 0.7,
|
| 326 |
+
"description": "SSSM shallow depth L=1 block (underfitting lower boundary check)"
|
| 327 |
+
},
|
| 328 |
+
{
|
| 329 |
+
"run_id": "Table2_Run6_SSSM_NoSkip",
|
| 330 |
+
"category": "Table2_Layer_Deduction",
|
| 331 |
+
"sequence_engine": "sssm",
|
| 332 |
+
"num_blocks": 3,
|
| 333 |
+
"use_skip": False,
|
| 334 |
+
"lambda_bio": 0.7,
|
| 335 |
+
"description": "SSSM without residual identity highway (testing gradient flow & vanishing dynamics)"
|
| 336 |
}
|
| 337 |
]
|
| 338 |
|
|
|
|
| 351 |
print(f"\n>>> Running [{run_idx + 1}/{len(runs_to_execute)}]: {run_config['run_id']}")
|
| 352 |
print(f" Description: {run_config['description']}")
|
| 353 |
|
| 354 |
+
num_blocks = run_config.get("num_blocks", 3)
|
| 355 |
+
use_skip = run_config.get("use_skip", True)
|
| 356 |
model = BioHCLSSSM_ExperimentModel(
|
| 357 |
sequence_engine=run_config['sequence_engine'],
|
| 358 |
+
num_blocks=num_blocks,
|
| 359 |
embed_dim=512,
|
| 360 |
+
d_state=64,
|
| 361 |
+
use_skip=use_skip
|
| 362 |
).to(DEVICE)
|
| 363 |
|
| 364 |
+
total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
| 365 |
+
params_m = round(total_params / 1e6, 2)
|
| 366 |
+
|
| 367 |
+
# Measure inference latency on test-sized dummy tensor
|
| 368 |
+
model.eval()
|
| 369 |
+
with torch.no_grad():
|
| 370 |
+
dummy_in = torch.randn(1, 3, 224, 224, device=DEVICE)
|
| 371 |
+
for _ in range(5):
|
| 372 |
+
_ = model(dummy_in)
|
| 373 |
+
if DEVICE.type == 'cuda':
|
| 374 |
+
torch.cuda.synchronize()
|
| 375 |
+
t_lat_start = time.time()
|
| 376 |
+
for _ in range(50):
|
| 377 |
+
_ = model(dummy_in)
|
| 378 |
+
if DEVICE.type == 'cuda':
|
| 379 |
+
torch.cuda.synchronize()
|
| 380 |
+
latency_ms = round((time.time() - t_lat_start) / 50 * 1000, 2)
|
| 381 |
+
|
| 382 |
optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-4)
|
| 383 |
scaler = torch.amp.GradScaler('cuda') if DEVICE.type == 'cuda' else None
|
| 384 |
|
|
|
|
| 464 |
"run_id": run_config['run_id'],
|
| 465 |
"category": run_config['category'],
|
| 466 |
"sequence_engine": run_config['sequence_engine'],
|
| 467 |
+
"num_blocks": num_blocks,
|
| 468 |
+
"use_skip": use_skip,
|
| 469 |
"lambda_bio": run_config['lambda_bio'],
|
| 470 |
+
"params_m": params_m,
|
| 471 |
+
"latency_ms": latency_ms,
|
| 472 |
"val_acc": best_metrics.get("val_acc", 0.0),
|
| 473 |
"val_f1": best_metrics.get("val_f1", 0.0),
|
| 474 |
"test_acc_pct": round(t_acc * 100, 2),
|