unknownBrain commited on
Commit
9c629b8
·
verified ·
1 Parent(s): f8615ba

Sync run_colab_remaining_ablations.py with 6 complete empirical runs and Tathagat Sir quotes

Browse files
Files changed (1) hide show
  1. run_colab_remaining_ablations.py +64 -11
run_colab_remaining_ablations.py CHANGED
@@ -245,11 +245,12 @@ class TransformerAttentionBlock(nn.Module):
245
  return self.norm2(x + ffn_out)
246
 
247
  class BioHCLSSSM_ExperimentModel(nn.Module):
248
- def __init__(self, sequence_engine="sssm", num_blocks=3, embed_dim=512, d_state=64):
249
  super().__init__()
250
  self.hier_cnn = HierarchicalConvBlock(3, embed_dim // 2)
251
  self.backbone = BackboneExtractor(embed_dim // 2)
252
  self.fusion = nn.Linear(embed_dim, embed_dim)
 
253
 
254
  self.blocks = nn.ModuleList()
255
  for i in range(num_blocks):
@@ -268,43 +269,70 @@ class BioHCLSSSM_ExperimentModel(nn.Module):
268
 
269
  h = tokens
270
  for block in self.blocks:
271
- h = block(h, h)
 
272
 
273
  feat = self.norm(h).mean(dim=1)
274
  logits = self.head(feat)
275
  return logits, feat
276
 
277
  # -------------------------------------------------------------
278
- # 4. EXECUTION MATRIX (4 CORE RUNS)
279
  # -------------------------------------------------------------
280
  runs_to_execute = [
281
  {
282
  "run_id": "Table3_Run1_lambda_0.2",
283
- "category": "Loss_Sensitivity",
284
  "sequence_engine": "sssm",
 
 
285
  "lambda_bio": 0.2,
286
  "description": "Bio-PINN loss weighting with light anatomical constraint (lambda=0.2)"
287
  },
288
  {
289
  "run_id": "Table3_Run2_lambda_0.5",
290
- "category": "Loss_Sensitivity",
291
  "sequence_engine": "sssm",
 
 
292
  "lambda_bio": 0.5,
293
  "description": "Bio-PINN loss weighting with moderate anatomical constraint (lambda=0.5)"
294
  },
295
  {
296
  "run_id": "Table3_Run3_lambda_1.0",
297
- "category": "Loss_Sensitivity",
298
  "sequence_engine": "sssm",
 
 
299
  "lambda_bio": 1.0,
300
- "description": "Bio-PINN loss weighting with over-weighted constraint (lambda=1.0)"
301
  },
302
  {
303
  "run_id": "Table1_Run4_MHSA_Attention",
304
- "category": "Component_Ablation",
305
  "sequence_engine": "attention",
 
 
306
  "lambda_bio": 0.7,
307
- "description": "SSSM replaced with Multi-Head Self-Attention (MHSA, 8 heads)"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
308
  }
309
  ]
310
 
@@ -323,13 +351,34 @@ for run_idx, run_config in enumerate(runs_to_execute):
323
  print(f"\n>>> Running [{run_idx + 1}/{len(runs_to_execute)}]: {run_config['run_id']}")
324
  print(f" Description: {run_config['description']}")
325
 
 
 
326
  model = BioHCLSSSM_ExperimentModel(
327
  sequence_engine=run_config['sequence_engine'],
328
- num_blocks=3,
329
  embed_dim=512,
330
- d_state=64
 
331
  ).to(DEVICE)
332
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
333
  optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-4)
334
  scaler = torch.amp.GradScaler('cuda') if DEVICE.type == 'cuda' else None
335
 
@@ -415,7 +464,11 @@ for run_idx, run_config in enumerate(runs_to_execute):
415
  "run_id": run_config['run_id'],
416
  "category": run_config['category'],
417
  "sequence_engine": run_config['sequence_engine'],
 
 
418
  "lambda_bio": run_config['lambda_bio'],
 
 
419
  "val_acc": best_metrics.get("val_acc", 0.0),
420
  "val_f1": best_metrics.get("val_f1", 0.0),
421
  "test_acc_pct": round(t_acc * 100, 2),
 
245
  return self.norm2(x + ffn_out)
246
 
247
  class BioHCLSSSM_ExperimentModel(nn.Module):
248
+ def __init__(self, sequence_engine="sssm", num_blocks=3, embed_dim=512, d_state=64, use_skip=True):
249
  super().__init__()
250
  self.hier_cnn = HierarchicalConvBlock(3, embed_dim // 2)
251
  self.backbone = BackboneExtractor(embed_dim // 2)
252
  self.fusion = nn.Linear(embed_dim, embed_dim)
253
+ self.use_skip = use_skip
254
 
255
  self.blocks = nn.ModuleList()
256
  for i in range(num_blocks):
 
269
 
270
  h = tokens
271
  for block in self.blocks:
272
+ res = h if self.use_skip else None
273
+ h = block(h, res)
274
 
275
  feat = self.norm(h).mean(dim=1)
276
  logits = self.head(feat)
277
  return logits, feat
278
 
279
  # -------------------------------------------------------------
280
+ # 4. EXECUTION MATRIX (6 COMPLETE RUNS FOR TABLES 1, 2, & 3)
281
  # -------------------------------------------------------------
282
  runs_to_execute = [
283
  {
284
  "run_id": "Table3_Run1_lambda_0.2",
285
+ "category": "Table3_Loss_Weighting",
286
  "sequence_engine": "sssm",
287
+ "num_blocks": 3,
288
+ "use_skip": True,
289
  "lambda_bio": 0.2,
290
  "description": "Bio-PINN loss weighting with light anatomical constraint (lambda=0.2)"
291
  },
292
  {
293
  "run_id": "Table3_Run2_lambda_0.5",
294
+ "category": "Table3_Loss_Weighting",
295
  "sequence_engine": "sssm",
296
+ "num_blocks": 3,
297
+ "use_skip": True,
298
  "lambda_bio": 0.5,
299
  "description": "Bio-PINN loss weighting with moderate anatomical constraint (lambda=0.5)"
300
  },
301
  {
302
  "run_id": "Table3_Run3_lambda_1.0",
303
+ "category": "Table3_Loss_Weighting",
304
  "sequence_engine": "sssm",
305
+ "num_blocks": 3,
306
+ "use_skip": True,
307
  "lambda_bio": 1.0,
308
+ "description": "Bio-PINN loss weighting with over-weighted constraint (lambda=1.0, over-regularization penalty test)"
309
  },
310
  {
311
  "run_id": "Table1_Run4_MHSA_Attention",
312
+ "category": "Table1_Component_Ablation",
313
  "sequence_engine": "attention",
314
+ "num_blocks": 3,
315
+ "use_skip": True,
316
  "lambda_bio": 0.7,
317
+ "description": "SSSM replaced with Multi-Head Self-Attention (MHSA, 8 heads, quadratic complexity benchmark)"
318
+ },
319
+ {
320
+ "run_id": "Table2_Run5_Depth_L1",
321
+ "category": "Table2_Layer_Deduction",
322
+ "sequence_engine": "sssm",
323
+ "num_blocks": 1,
324
+ "use_skip": True,
325
+ "lambda_bio": 0.7,
326
+ "description": "SSSM shallow depth L=1 block (underfitting lower boundary check)"
327
+ },
328
+ {
329
+ "run_id": "Table2_Run6_SSSM_NoSkip",
330
+ "category": "Table2_Layer_Deduction",
331
+ "sequence_engine": "sssm",
332
+ "num_blocks": 3,
333
+ "use_skip": False,
334
+ "lambda_bio": 0.7,
335
+ "description": "SSSM without residual identity highway (testing gradient flow & vanishing dynamics)"
336
  }
337
  ]
338
 
 
351
  print(f"\n>>> Running [{run_idx + 1}/{len(runs_to_execute)}]: {run_config['run_id']}")
352
  print(f" Description: {run_config['description']}")
353
 
354
+ num_blocks = run_config.get("num_blocks", 3)
355
+ use_skip = run_config.get("use_skip", True)
356
  model = BioHCLSSSM_ExperimentModel(
357
  sequence_engine=run_config['sequence_engine'],
358
+ num_blocks=num_blocks,
359
  embed_dim=512,
360
+ d_state=64,
361
+ use_skip=use_skip
362
  ).to(DEVICE)
363
 
364
+ total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
365
+ params_m = round(total_params / 1e6, 2)
366
+
367
+ # Measure inference latency on test-sized dummy tensor
368
+ model.eval()
369
+ with torch.no_grad():
370
+ dummy_in = torch.randn(1, 3, 224, 224, device=DEVICE)
371
+ for _ in range(5):
372
+ _ = model(dummy_in)
373
+ if DEVICE.type == 'cuda':
374
+ torch.cuda.synchronize()
375
+ t_lat_start = time.time()
376
+ for _ in range(50):
377
+ _ = model(dummy_in)
378
+ if DEVICE.type == 'cuda':
379
+ torch.cuda.synchronize()
380
+ latency_ms = round((time.time() - t_lat_start) / 50 * 1000, 2)
381
+
382
  optimizer = optim.AdamW(model.parameters(), lr=5e-4, weight_decay=1e-4)
383
  scaler = torch.amp.GradScaler('cuda') if DEVICE.type == 'cuda' else None
384
 
 
464
  "run_id": run_config['run_id'],
465
  "category": run_config['category'],
466
  "sequence_engine": run_config['sequence_engine'],
467
+ "num_blocks": num_blocks,
468
+ "use_skip": use_skip,
469
  "lambda_bio": run_config['lambda_bio'],
470
+ "params_m": params_m,
471
+ "latency_ms": latency_ms,
472
  "val_acc": best_metrics.get("val_acc", 0.0),
473
  "val_f1": best_metrics.get("val_f1", 0.0),
474
  "test_acc_pct": round(t_acc * 100, 2),