unknownBrain's picture
Complete medical imaging workspace dump (weights, deliverables, papers, notebooks, code, figures)
eded815 verified
Raw History Blame Contribute Delete
47 kB
!pip install -q kagglehub wandb huggingface_hub timm albumentations torchvision scikit-learn seaborn # ============ HARD-CODE YOUR TOKENS / CONFIG HERE ============ HF_TOKEN = os.environ.get('HF_TOKEN', '') # <-- paste your Hugging Face WRITE token WANDB_API_KEY = "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" # <-- paste your W&B API key (leave "" to get an interactive login prompt) HF_USERNAME = "unknownBrain" # <-- change this HF_REPO_ID = f"{HF_USERNAME}/chest-xray-pneumonia-sssm-ablation-bioprior" # HF model repo that will be created/used # ================================================================ import os from huggingface_hub import login as hf_login, HfApi os.environ["HF_TOKEN"] = HF_TOKEN hf_login(token=HF_TOKEN) hf_api = HfApi() print("βœ… Logged in to Hugging Face Hub") import wandb if WANDB_API_KEY: wandb.login(key=WANDB_API_KEY) else: wandb.login() print("βœ… Logged in to Weights & Biases") import os, glob, random, math import numpy as np import pandas as pd import cv2 from PIL import Image from tqdm import tqdm import torch import torch.nn as nn import torch.nn.functional as F import torch.optim as optim from torch.optim.lr_scheduler import CosineAnnealingLR from torch.utils.data import DataLoader, Dataset, WeightedRandomSampler from sklearn.model_selection import train_test_split from sklearn.metrics import (classification_report, confusion_matrix, roc_curve, auc, precision_recall_fscore_support, accuracy_score) import matplotlib.pyplot as plt import seaborn as sns import timm import albumentations as A from albumentations.pytorch import ToTensorV2 def get_safe_device(): return torch.device("cuda" if torch.cuda.is_available() else "cpu") DEVICE = get_safe_device() print(f"βœ… Training on: {DEVICE}") SEED = 42 random.seed(SEED); np.random.seed(SEED) torch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED) import kagglehub DATA_DIR = kagglehub.dataset_download("paultimothymooney/chest-xray-pneumonia") print("Dataset downloaded to:", DATA_DIR) for root, dirs, files in os.walk(DATA_DIR): depth = root[len(DATA_DIR):].count(os.sep) if depth <= 2: print(root) CLASS_NAMES = ["NORMAL", "PNEUMONIA"] NUM_CLASSES = len(CLASS_NAMES) LABEL_MAP = {name: idx for idx, name in enumerate(CLASS_NAMES)} def find_chest_xray_root(base_dir): """kagglehub extracts a nested folder; locate the level containing train/val/test.""" for root, dirs, files in os.walk(base_dir): if "train" in dirs and "test" in dirs: return root return base_dir ROOT_DIR = find_chest_xray_root(DATA_DIR) print("Dataset root:", ROOT_DIR) def gather_split(split_name): rows = [] for cls in CLASS_NAMES: folder = os.path.join(ROOT_DIR, split_name, cls) if not os.path.isdir(folder): continue paths = (glob.glob(os.path.join(folder, "*.jpeg")) + glob.glob(os.path.join(folder, "*.jpg")) + glob.glob(os.path.join(folder, "*.png"))) for p in paths: rows.append({"path": p, "label": cls}) return pd.DataFrame(rows) df_train_raw = gather_split("train") df_val_raw = gather_split("val") # tiny/broken split in the original dataset df_test = gather_split("test") # kept separate, untouched # ---- FIX: combine train + val, then re-split with stratification ---- df_trainval = pd.concat([df_train_raw, df_val_raw], ignore_index=True) print(f"Combined train+val pool: {len(df_trainval)}") print(df_trainval["label"].value_counts()) print(f"\nHeld-out TEST set (untouched): {len(df_test)}") print(df_test["label"].value_counts()) df_train, df_val = train_test_split( df_trainval, test_size=0.2, stratify=df_trainval["label"], random_state=SEED, ) df_train = df_train.reset_index(drop=True) df_val = df_val.reset_index(drop=True) print(f"\nβœ… New stratified split -> Train: {len(df_train)} | Val: {len(df_val)} | Test: {len(df_test)}") print("Train distribution:\n", df_train["label"].value_counts()) print("Val distribution:\n", df_val["label"].value_counts()) class_counts = df_train["label"].value_counts().to_dict() print("Train class counts:", class_counts) # Inverse-frequency class weights, used inside CrossEntropyLoss for both baseline & BioHCL-SSSM total = sum(class_counts.values()) class_weights_list = [total / (NUM_CLASSES * class_counts[c]) for c in CLASS_NAMES] CLASS_WEIGHTS = torch.tensor(class_weights_list, dtype=torch.float32).to(DEVICE) print("CrossEntropyLoss class weights:", CLASS_WEIGHTS) # Per-sample weights for WeightedRandomSampler (oversamples the minority NORMAL class each epoch) sample_weights = df_train["label"].map(lambda c: 1.0 / class_counts[c]).values sample_weights = torch.DoubleTensor(sample_weights) IMG_SIZE = 224 IMAGENET_MEAN = [0.485, 0.456, 0.406] IMAGENET_STD = [0.229, 0.224, 0.225] train_transform = A.Compose([ A.Resize(IMG_SIZE, IMG_SIZE), A.HorizontalFlip(p=0.5), A.Rotate(limit=10, p=0.5, border_mode=cv2.BORDER_CONSTANT), A.ShiftScaleRotate(shift_limit=0.05, scale_limit=0.05, rotate_limit=0, p=0.3, border_mode=cv2.BORDER_CONSTANT), A.RandomBrightnessContrast(brightness_limit=0.15, contrast_limit=0.15, p=0.3), A.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), ToTensorV2(), ]) eval_transform = A.Compose([ A.Resize(IMG_SIZE, IMG_SIZE), A.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), ToTensorV2(), ]) class ChestXrayDataset(Dataset): def __init__(self, df, transform=None): self.df = df.reset_index(drop=True) self.transform = transform def __len__(self): return len(self.df) def __getitem__(self, idx): path = self.df.iloc[idx]["path"] label = LABEL_MAP[self.df.iloc[idx]["label"]] img = Image.open(path).convert("RGB") img = np.array(img) if self.transform: img = self.transform(image=img)["image"] return img, label BATCH_SIZE = 8 def create_dataloaders(batch_size=BATCH_SIZE): train_ds = ChestXrayDataset(df_train, transform=train_transform) val_ds = ChestXrayDataset(df_val, transform=eval_transform) test_ds = ChestXrayDataset(df_test, transform=eval_transform) sampler = WeightedRandomSampler(sample_weights, num_samples=len(sample_weights), replacement=True) train_loader = DataLoader(train_ds, batch_size=batch_size, sampler=sampler, num_workers=2, pin_memory=True) val_loader = DataLoader(val_ds, batch_size=batch_size, shuffle=False, num_workers=2, pin_memory=True) test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=2, pin_memory=True) print(f"Train: {len(train_ds)} | Val: {len(val_ds)} | Test: {len(test_ds)}") return train_loader, val_loader, test_loader train_loader, val_loader, test_loader = create_dataloaders() USE_GRAD_CHECKPOINT = True # off by default now that embed_dim=256 + fewer blocks already fixed the OOM; flip to True only if needed class ResidualConnector(nn.Module): def __init__(self, embed_dim, skip_step=2): super().__init__() self.skip_step = skip_step self.proj = nn.Linear(embed_dim, embed_dim) self.norm = nn.LayerNorm(embed_dim) def forward(self, x, residuals): skip = residuals.get("skip", torch.zeros_like(x)) return self.norm(x + self.proj(skip)) class BackboneExtractor(nn.Module): BACKBONES = { "densenet": lambda: timm.create_model("densenet121", pretrained=True, num_classes=0, global_pool="avg"), "swin": lambda: timm.create_model("swin_tiny_patch4_window7_224", pretrained=True, num_classes=0), "vit": lambda: timm.create_model("vit_base_patch16_224", pretrained=True, num_classes=0), } def __init__(self, backbone_type="densenet", embed_dim=512, dropout=0.1): super().__init__() self.backbone = self.BACKBONES[backbone_type]() backbone_dim = {"densenet": 1024, "swin": 768, "vit": 768}[backbone_type] self.proj = nn.Linear(backbone_dim, embed_dim) self.dropout = nn.Dropout(dropout) def forward(self, x): feats = self.backbone(x) if feats.dim() == 3: feats = feats.mean(dim=1) return self.dropout(self.proj(feats)) class QuadraticPositionalEncoder(nn.Module): def __init__(self, embed_dim, num_levels=3): super().__init__() self.num_levels = num_levels self.proj = nn.Linear(embed_dim, embed_dim * num_levels) def forward(self, x): B, N, C = x.shape feat_proj = self.proj(x).view(B, N, self.num_levels, -1) pos_enc = torch.zeros_like(feat_proj) for lvl in range(self.num_levels): lp = feat_proj[:, :, lvl, :] pos_enc[:, :, lvl, :] = lp**2 * torch.sin(lvl * torch.pi * lp) return pos_enc.mean(dim=2) class HierarchicalConvBlock(nn.Module): def __init__(self, in_channels, embed_dim, kernel_sizes=[3,5,7]): super().__init__() self.per_conv = embed_dim // len(kernel_sizes) self.convs = nn.ModuleList([ nn.Conv2d(in_channels, self.per_conv, k, padding=k//2) for k in kernel_sizes ]) actual = self.per_conv * len(kernel_sizes) self.fusion = nn.Conv2d(actual, embed_dim, 1) def forward(self, x): feats = [F.adaptive_avg_pool2d(conv(x), (8, 8)) for conv in self.convs] # 8x8=64 tokens instead of 16x16=256 -> ~4x less recurrent SSM memory return self.fusion(torch.cat(feats, dim=1)).flatten(2).transpose(1, 2) class LinearBioSSSMBlock(nn.Module): def __init__(self, d_model, d_state=16, expand=2): super().__init__() self.d_inner = int(expand * d_model) self.d_state = d_state self.in_proj = nn.Linear(d_model, self.d_inner * 2) self.bio_proj = nn.Linear(d_model, 1) self.x_proj = nn.Linear(self.d_inner, d_state * 2 + 1) self.dt_proj = nn.Linear(1, self.d_inner) self.A_log = nn.Parameter(torch.log(torch.arange(1, d_state + 1).float())) self.D = nn.Parameter(torch.ones(self.d_inner)) self.out_proj = nn.Linear(self.d_inner, d_model) self.norm = nn.LayerNorm(d_model) def forward(self, x, residuals=None): # Runs recurrent state accumulation in float32 to prevent FP16 overflow (>65504) with torch.autocast(device_type=x.device.type, enabled=False): x_f = x.float() B, L, D = x_f.shape bio_gate = torch.sigmoid(self.bio_proj(x_f.mean(dim=1, keepdim=True))) xz = self.in_proj(x_f) x_in, z = xz.chunk(2, dim=-1) x_in = F.silu(x_in) ssm = self.x_proj(x_in) dt_raw = ssm[:, :, :1] B_mat = ssm[:, :, 1:1+self.d_state] C_mat = ssm[:, :, 1+self.d_state:] dt = F.softplus(self.dt_proj(dt_raw)) A = -torch.exp(self.A_log.float()) h = torch.zeros(B, self.d_inner, self.d_state, device=x.device, dtype=torch.float32) ys = [] for t in range(L): dA = torch.exp(dt[:, t, :, None] * A[None, None, :]) dB = dt[:, t, :, None] * B_mat[:, t, None, :] h = h * dA + dB * x_in[:, t, :, None] y_t = (h * C_mat[:, t, None, :]).sum(-1) + self.D.float() * x_in[:, t, :] ys.append(y_t) out = torch.stack(ys, dim=1) out = out * torch.sigmoid(z) * bio_gate out = self.out_proj(out) res_val = residuals.float() if residuals is not None else 0 out = self.norm(out + res_val) return out.to(x.dtype) class DualChainExtractor(nn.Module): def __init__(self, embed_dim, backbone_type="densenet"): super().__init__() self.hier_cnn = HierarchicalConvBlock(3, embed_dim//2) self.backbone = BackboneExtractor(backbone_type, embed_dim//2) self.fusion = nn.Linear(embed_dim, embed_dim) self.pos_enc = QuadraticPositionalEncoder(embed_dim) def forward(self, x): cnn_feats = self.hier_cnn(x) bb_feats = self.backbone(x).unsqueeze(1).expand(-1, cnn_feats.size(1), -1) fused = self.fusion(torch.cat([cnn_feats, bb_feats], dim=-1)) return fused + self.pos_enc(fused) class SingleChainExtractor(nn.Module): def __init__(self, embed_dim, mode="hier_cnn", backbone_type="densenet"): super().__init__() self.extractor = HierarchicalConvBlock(3, embed_dim) if mode == "hier_cnn" \ else BackboneExtractor(backbone_type, embed_dim) self.pos_enc = QuadraticPositionalEncoder(embed_dim) def forward(self, x): feats = self.extractor(x) return feats + self.pos_enc(feats) class BioHCLSSSM(nn.Module): def __init__(self, num_classes=NUM_CLASSES, embed_dim=512, chain_mode="dual", backbone_type="densenet", num_sssm_blocks=4, skip_step=2): super().__init__() self.chain_mode = chain_mode self.num_blocks = num_sssm_blocks if chain_mode == "dual": self.extractor = DualChainExtractor(embed_dim, backbone_type) else: mode = "hier_cnn" if chain_mode == "single_cnn" else "backbone" self.extractor = SingleChainExtractor(embed_dim, mode, backbone_type) self.sssm_blocks = nn.ModuleList([ LinearBioSSSMBlock(embed_dim, d_state=16+i*4) for i in range(num_sssm_blocks) ]) self.res_connectors = nn.ModuleList([ ResidualConnector(embed_dim, skip_step) for _ in range(num_sssm_blocks) ]) self.norm = nn.LayerNorm(embed_dim) self.head = nn.Linear(embed_dim, num_classes) self.bio_prior = nn.Parameter(torch.randn(1, embed_dim) * 0.1) def forward(self, x): tokens = self.extractor(x) residuals = {} block_outs = [tokens] for i, (block, res_conn) in enumerate(zip(self.sssm_blocks, self.res_connectors)): prev_out = block_outs[-1] res_input = residuals.get("skip", prev_out) if self.training and USE_GRAD_CHECKPOINT: # gradient checkpointing: recompute the 64-step recurrence on the backward pass # instead of keeping every timestep's activations in memory. Costs ~1.5-2x compute time. # Only turn this on if you OOM again after the embed_dim/block-count reduction below. out = torch.utils.checkpoint.checkpoint(block, prev_out, res_input, use_reentrant=False) else: out = block(prev_out, res_input) residuals["skip"] = block_outs[i - res_conn.skip_step] if i >= res_conn.skip_step else prev_out block_outs.append(out) x = self.norm(block_outs[-1]) # [ABLATION: bio_prior removed] global_feat = x.mean(dim=1) return self.head(global_feat), global_feat hf_api.create_repo(repo_id=HF_REPO_ID, repo_type="model", exist_ok=True, private=False) print(f"βœ… HF repo ready: https://huggingface.co/{HF_REPO_ID}") def upload_to_hf(local_path, path_in_repo=None): """Upload a local file (checkpoint or CSV) to the HF model repo.""" path_in_repo = path_in_repo or os.path.basename(local_path) hf_api.upload_file( path_or_fileobj=local_path, path_in_repo=path_in_repo, repo_id=HF_REPO_ID, repo_type="model", ) print(f" ⬆️ Uploaded {local_path} -> hf://{HF_REPO_ID}/{path_in_repo}") EPOCHS_BASELINE = 20 EPOCHS_KD = 20 LR_BASELINE = 1e-4 LR_TEACHER = 1e-4 LR_STUDENT = 1e-4 LAMBDA_PHYS = 0.0 # was 0.3 -- diagnostics showed this term driving feature collapse # (avg_phys crashed 0.60 -> 0.004 while avg_ce sat pinned at ln(2)=0.693, # i.e. the model learned to make every image's features identical # rather than learn to classify). Set to 0 to confirm; can reintroduce # a small value (e.g. 0.02) later once teacher is confirmed learning normally. KD_TEMP = 4.0 KD_ALPHA = 0.5 run = wandb.init( project="chest-xray-pneumonia-ablation-bioprior", config={ "dataset": "paultimothymooney/chest-xray-pneumonia", "img_size": IMG_SIZE, "batch_size": BATCH_SIZE, "epochs_baseline": EPOCHS_BASELINE, "epochs_kd": EPOCHS_KD, "lr_baseline": LR_BASELINE, "lr_teacher": LR_TEACHER, "lr_student": LR_STUDENT, "lambda_phys": LAMBDA_PHYS, "kd_temp": KD_TEMP, "kd_alpha": KD_ALPHA, "class_weights": class_weights_list, "architecture": "BioHCL-SSSM ABLATION [bio_prior removed] (teacher: 3 blocks, student: 2 blocks) + ResNet50 baseline", }, ) print("βœ… W&B run:", run.url) def get_resnet50(num_classes=NUM_CLASSES): m = timm.create_model("resnet50", pretrained=True, num_classes=num_classes) return m.to(DEVICE) def compute_metrics(labels, preds): acc = accuracy_score(labels, preds) precision, recall, f1, _ = precision_recall_fscore_support( labels, preds, average="binary", pos_label=LABEL_MAP["PNEUMONIA"], zero_division=0 ) return acc, precision, recall, f1 def baseline_train(model, train_loader, val_loader, epochs=EPOCHS_BASELINE): optimizer = optim.AdamW(model.parameters(), lr=LR_BASELINE, weight_decay=1e-4) scheduler = CosineAnnealingLR(optimizer, T_max=epochs) criterion = nn.CrossEntropyLoss() best_f1 = 0.0 best_state = {"preds": [], "labels": [], "probs": []} for epoch in range(epochs): model.train() running_loss, correct, total = 0.0, 0, 0 for images, labels in tqdm(train_loader, desc=f"[Baseline] E{epoch+1}", leave=False): images, labels = images.to(DEVICE), labels.to(DEVICE) optimizer.zero_grad() logits = model(images) loss = criterion(logits, labels) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) optimizer.step() running_loss += loss.item() * labels.size(0) correct += (logits.argmax(1) == labels).sum().item() total += labels.size(0) scheduler.step() train_loss = running_loss / total train_acc = correct / total model.eval() val_correct, val_total, val_loss_sum = 0, 0, 0.0 all_preds, all_labels, all_probs = [], [], [] with torch.no_grad(): for images, labels in val_loader: images, labels = images.to(DEVICE), labels.to(DEVICE) logits = model(images) val_loss_sum += criterion(logits, labels).item() * labels.size(0) probs = F.softmax(logits, dim=1) pred = probs.argmax(1) val_correct += (pred == labels).sum().item() val_total += labels.size(0) all_preds.extend(pred.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) all_probs.extend(probs.cpu().numpy()) val_loss = val_loss_sum / val_total val_acc, val_prec, val_rec, val_f1 = compute_metrics(all_labels, all_preds) wandb.log({ "epoch": epoch + 1, "baseline/train_loss": train_loss, "baseline/train_acc": train_acc, "baseline/val_loss": val_loss, "baseline/val_acc": val_acc, "baseline/val_precision": val_prec, "baseline/val_recall": val_rec, "baseline/val_f1": val_f1, }) print(f"[Baseline] E{epoch+1:2d}: TrainAcc={train_acc*100:.2f}% " f"ValAcc={val_acc*100:.2f}% F1={val_f1:.4f} (Best F1={best_f1:.4f})") if val_f1 > best_f1: best_f1 = val_f1 best_state = {"preds": all_preds, "labels": all_labels, "probs": all_probs} torch.save(model.state_dict(), "best_baseline.pt") upload_to_hf("best_baseline.pt") return best_f1, best_state resnet50 = get_resnet50() print(f"ResNet50 params: {sum(p.numel() for p in resnet50.parameters())/1e6:.1f}M") baseline_best_f1, baseline_best_state = baseline_train(resnet50, train_loader, val_loader) print(f"\nπŸ† Baseline best Val F1: {baseline_best_f1:.4f}") import gc # Free the baseline ResNet50 from GPU before allocating teacher+student β€” # two BioHCL-SSSM models training simultaneously need all the headroom they can get. resnet50 = resnet50.to("cpu") del resnet50 gc.collect() torch.cuda.empty_cache() print(f"GPU memory allocated after cleanup: {torch.cuda.memory_allocated()/1e9:.2f} GB") def bio_pinn_loss(model, images, labels, lambda_phys=LAMBDA_PHYS): logits, features = model(images) # Same fp16-overflow risk as the SSM recurrence -- force this in fp32 too. with torch.autocast(device_type=images.device.type, enabled=False): ce_loss = F.cross_entropy(logits.float(), labels) phys_loss = torch.mean(torch.norm(features.float().roll(1, dims=0) - features.float(), dim=-1)) return {"ce": ce_loss, "phys": phys_loss, "total": ce_loss + lambda_phys * phys_loss}, logits def kd_loss(logits_s, logits_t, temp=KD_TEMP, alpha=KD_ALPHA): # Runs KL divergence in float32, detaches teacher graph, and clamps near-zero probabilities with torch.autocast(device_type=logits_s.device.type, enabled=False): logits_s_f = logits_s.float() logits_t_f = logits_t.float().detach() soft_s = F.log_softmax(logits_s_f / temp, dim=1) soft_t = F.softmax(logits_t_f / temp, dim=1) soft_t = torch.clamp(soft_t, min=1e-5, max=1.0 - 1e-5) loss = F.kl_div(soft_s, soft_t, reduction="batchmean") * (temp ** 2) * alpha return loss def train_teacher_epoch(model, dataloader, optimizer, scaler, epoch): model.train() total_loss, correct, total = 0.0, 0, 0 ce_sum, phys_sum, skipped_steps, n_batches = 0.0, 0.0, 0, 0 pbar = tqdm(dataloader, desc=f"[Teacher] E{epoch+1}", leave=False) for images, labels in pbar: images, labels = images.to(DEVICE), labels.to(DEVICE) optimizer.zero_grad() # AMP Autocast with torch.amp.autocast('cuda'): loss_dict, logits = bio_pinn_loss(model, images, labels) scale_before = scaler.get_scale() scaler.scale(loss_dict["total"]).backward() scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0) scaler.step(optimizer) scaler.update() # If GradScaler saw inf/nan gradients, it skips optimizer.step() and lowers the scale -- # a drop here means this batch's update never actually happened. if scaler.get_scale() < scale_before: skipped_steps += 1 total_loss += loss_dict["total"].item() * labels.size(0) ce_sum += loss_dict["ce"].item() * labels.size(0) phys_sum += loss_dict["phys"].item() * labels.size(0) n_batches += 1 correct += (logits.argmax(1) == labels).sum().item() total += labels.size(0) pbar.set_postfix({"Loss": f"{loss_dict['total'].item():.3f}", "Acc": f"{100*correct/total:.1f}%"}) print(f" [diag] avg_ce={ce_sum/total:.4f} avg_phys={phys_sum/total:.4f} " f"skipped_steps={skipped_steps}/{n_batches} final_scaler_scale={scaler.get_scale():.1f}") return total_loss / total, correct / total def train_student_epoch(student, teacher, dataloader, optimizer, scaler, epoch): student.train() teacher.eval() total_loss, correct, total = 0.0, 0, 0 ce_sum, kd_sum = 0.0, 0.0 skipped_steps, n_batches = 0, 0 pred_class_counts = {0: 0, 1: 0} teacher_class_counts = {0: 0, 1: 0} pbar = tqdm(dataloader, desc=f"[Student] E{epoch+1}", leave=False) for images, labels in pbar: images = images.to(DEVICE) labels = labels.to(DEVICE) optimizer.zero_grad() # ---------------- AMP Forward ---------------- with torch.amp.autocast("cuda"): logits_s, _ = student(images) if torch.isnan(logits_s).any(): raise RuntimeError("NaN detected in student logits.") if torch.isinf(logits_s).any(): raise RuntimeError("Inf detected in student logits.") with torch.no_grad(): logits_t, _ = teacher(images) # ---------------- FP32 Loss ---------------- with torch.autocast(device_type="cuda", enabled=False): ce_part = F.cross_entropy( logits_s.float(), labels ) kd_part = kd_loss( logits_s, logits_t ) loss = ce_part + 0.2 * kd_part scale_before = scaler.get_scale() scaler.scale(loss).backward() scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_( student.parameters(), 1.0 ) scaler.step(optimizer) scaler.update() if scaler.get_scale() < scale_before: skipped_steps += 1 total_loss += loss.item() * labels.size(0) ce_sum += ce_part.item() * labels.size(0) kd_sum += kd_part.item() * labels.size(0) n_batches += 1 preds = logits_s.argmax(1) t_preds = logits_t.argmax(1) for c in (0, 1): pred_class_counts[c] += (preds == c).sum().item() teacher_class_counts[c] += (t_preds == c).sum().item() correct += (preds == labels).sum().item() total += labels.size(0) pbar.set_postfix( { "Loss": f"{loss.item():.3f}", "Acc": f"{100*correct/total:.1f}%" } ) print( f" [diag] avg_ce={ce_sum/total:.4f} " f"avg_kd={kd_sum/total:.4f} " f"skipped_steps={skipped_steps}/{n_batches} " f"final_scaler_scale={scaler.get_scale():.1f}" ) print( f" [diag] student pred distribution: {pred_class_counts} " f"teacher pred distribution (same batches): {teacher_class_counts} " f"(0={CLASS_NAMES[0]}, 1={CLASS_NAMES[1]})" ) return total_loss / total, correct / total def validate(model, dataloader): model.eval() total_loss, total = 0.0, 0 all_preds, all_labels, all_probs = [], [], [] with torch.no_grad(): for images, labels in dataloader: images, labels = images.to(DEVICE), labels.to(DEVICE) with torch.amp.autocast('cuda'): logits, _ = model(images) loss = F.cross_entropy(logits.float(), labels) total_loss += loss.item() * labels.size(0) probs = F.softmax(logits, dim=1) pred = probs.argmax(1) all_preds.extend(pred.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) all_probs.extend(probs.cpu().numpy()) total += labels.size(0) val_loss = total_loss / total acc, prec, rec, f1 = compute_metrics(all_labels, all_preds) return val_loss, acc, prec, rec, f1, all_preds, all_labels, all_probs RESUME_CHECKPOINT_NAME = "kd_resume_checkpoint_bioprior.pt" def try_resume_from_hf(teacher, student, teacher_opt, student_opt, teacher_sched, student_sched, scaler_t, scaler_s): """If a previous run got cut off (Colab disconnect), pick up where it left off. Checkpoint lives on the HF repo, not local disk, so it survives a fresh runtime.""" try: from huggingface_hub import hf_hub_download ckpt_path = hf_hub_download(repo_id=HF_REPO_ID, filename=RESUME_CHECKPOINT_NAME) ckpt = torch.load(ckpt_path, map_location=DEVICE) teacher.load_state_dict(ckpt["teacher_state"]) student.load_state_dict(ckpt["student_state"]) teacher_opt.load_state_dict(ckpt["teacher_opt_state"]) student_opt.load_state_dict(ckpt["student_opt_state"]) teacher_sched.load_state_dict(ckpt["teacher_sched_state"]) student_sched.load_state_dict(ckpt["student_sched_state"]) scaler_t.load_state_dict(ckpt["scaler_t_state"]) scaler_s.load_state_dict(ckpt["scaler_s_state"]) epoch_num = ckpt['epoch'] + 1 f1_so_far = ckpt['best_val_f1'] print(f"βœ… Resumed from checkpoint at epoch {epoch_num}, best F1 so far: {f1_so_far:.4f}") return ckpt["epoch"] + 1, ckpt["best_val_f1"] except Exception as e: print(f"No resumable checkpoint found ({type(e).__name__}) β€” starting fresh.") return 0, 0.0 def train_chest_xray(epochs=EPOCHS_KD): print("\nπŸš€ BioHCL-SSSM Teacher -> Student β€” Chest X-Ray Pneumonia") # Clean memory before model allocation torch.cuda.empty_cache() # embed_dim 512->256 and fewer SSSM blocks: the recurrent scan cost scales with both, # this is the single biggest lever on wall-clock time (bigger than batch size). teacher = BioHCLSSSM(num_classes=NUM_CLASSES, embed_dim=256, chain_mode="dual", backbone_type="densenet", num_sssm_blocks=3).to(DEVICE) student = BioHCLSSSM(num_classes=NUM_CLASSES, embed_dim=256, chain_mode="dual", backbone_type="densenet", num_sssm_blocks=2).to(DEVICE) print(f"Teacher: {sum(p.numel() for p in teacher.parameters())/1e6:.1f}M params") print(f"Student: {sum(p.numel() for p in student.parameters())/1e6:.1f}M params") teacher_opt = optim.AdamW(teacher.parameters(), lr=LR_TEACHER, weight_decay=1e-4) student_opt = optim.AdamW(student.parameters(), lr=LR_STUDENT, weight_decay=1e-4) teacher_sched = CosineAnnealingLR(teacher_opt, T_max=epochs) student_sched = CosineAnnealingLR(student_opt, T_max=epochs) # Initialize PyTorch AMP Scalers scaler_t = torch.amp.GradScaler('cuda') scaler_s = torch.amp.GradScaler('cuda') start_epoch, best_val_f1 = try_resume_from_hf( teacher, student, teacher_opt, student_opt, teacher_sched, student_sched, scaler_t, scaler_s ) best_state = {"preds": [], "labels": [], "probs": []} for epoch in range(start_epoch, epochs): t_loss, t_acc = train_teacher_epoch(teacher, train_loader, teacher_opt, scaler_t, epoch) teacher_sched.step() s_loss, s_train_acc = train_student_epoch(student, teacher, train_loader, student_opt, scaler_s, epoch) s_val_loss, s_val_acc, s_prec, s_rec, s_f1, preds, labels, probs = validate(student, val_loader) student_sched.step() # Release cached GPU memory torch.cuda.empty_cache() wandb.log({ "epoch": epoch + 1, "teacher/train_loss": t_loss, "teacher/train_acc": t_acc, "student/train_loss": s_loss, "student/train_acc": s_train_acc, "student/val_loss": s_val_loss, "student/val_acc": s_val_acc, "student/val_precision": s_prec, "student/val_recall": s_rec, "student/val_f1": s_f1, }) print(f"E{epoch+1:2d}: Teacher TrainAcc={t_acc*100:.2f}% | " f"Student TrainAcc={s_train_acc*100:.2f}% ValAcc={s_val_acc*100:.2f}% " f"F1={s_f1:.4f} (Best F1={best_val_f1:.4f})") if s_f1 > best_val_f1: best_val_f1 = s_f1 best_state = {"preds": preds, "labels": labels, "probs": probs} torch.save(student.state_dict(), "best_student_bioprior.pt") upload_to_hf("best_student_bioprior.pt") # Save a full resume checkpoint every epoch (not just on new best) so a Colab # disconnect only costs you the current epoch, not the whole run. torch.save({ "epoch": epoch, "best_val_f1": best_val_f1, "teacher_state": teacher.state_dict(), "student_state": student.state_dict(), "teacher_opt_state": teacher_opt.state_dict(), "student_opt_state": student_opt.state_dict(), "teacher_sched_state": teacher_sched.state_dict(), "student_sched_state": student_sched.state_dict(), "scaler_t_state": scaler_t.state_dict(), "scaler_s_state": scaler_s.state_dict(), }, RESUME_CHECKPOINT_NAME) upload_to_hf(RESUME_CHECKPOINT_NAME) return teacher, student, best_val_f1, best_state teacher, student, student_best_f1, student_best_state = train_chest_xray() print(f"\nπŸ† Student best Val F1: {student_best_f1:.4f}") # ============================================================ # EVALUATION & XAI FOR ABLATION MODEL (bio_prior removed) # ============================================================ from huggingface_hub import hf_hub_download import numpy as np import pandas as pd import torch import torch.nn as nn import torch.nn.functional as F import matplotlib.pyplot as plt import seaborn as sns import cv2 import shap from sklearn.metrics import ( classification_report, confusion_matrix, roc_curve, auc, precision_recall_curve, average_precision_score, ) import wandb import os # ============================================================ # REPO ID FOR ABLATION MODEL (bio_prior removed) # ============================================================ REPO_ID = "unknownBrain/chest-xray-pneumonia-sssm-ablation-bioprior" print("\n" + "="*60) print("πŸ“₯ LOADING ABLATION MODEL FROM HUGGING FACE") print("="*60) # Recreate the student model (same as trained in notebook) # Note: This matches the architecture from the ablation notebook student = BioHCLSSSM( num_classes=NUM_CLASSES, embed_dim=256, chain_mode="dual", backbone_type="densenet", num_sssm_blocks=2 ).to(DEVICE) # Download checkpoint from HF student_weights_path = hf_hub_download( repo_id=REPO_ID, filename="best_student_bioprior.pt" ) student.load_state_dict(torch.load(student_weights_path, map_location=DEVICE)) student.eval() print(f"βœ… Model loaded from: {REPO_ID}") print(f"βœ… Checkpoint: best_student_bioprior.pt") # ============================================================ # EVALUATION FUNCTION (NO BASELINE) # ============================================================ def evaluate_model(model, name, test_loader, model_type="biohclsssm"): model.eval() all_preds, all_labels, all_probs = [], [], [] with torch.no_grad(): for images, labels in test_loader: images = images.to(DEVICE) if model_type == "biohclsssm": logits, _ = model(images) else: logits = model(images) probs = F.softmax(logits, dim=1) pred = probs.argmax(1) all_preds.extend(pred.cpu().numpy()) all_labels.extend(labels.numpy()) all_probs.extend(probs.cpu().numpy()) all_probs = np.array(all_probs) acc, prec, rec, f1 = compute_metrics(all_labels, all_preds) print(f"\n{'='*50}") print(f"πŸ† {name} β€” TEST Accuracy: {acc*100:.2f}% | F1: {f1:.4f}") print(classification_report(all_labels, all_preds, target_names=CLASS_NAMES)) cm = confusion_matrix(all_labels, all_preds) plt.figure(figsize=(5, 4)) sns.heatmap(cm, annot=True, fmt="d", cmap="Blues", xticklabels=CLASS_NAMES, yticklabels=CLASS_NAMES) plt.title(f"{name} β€” Test Confusion Matrix") plt.ylabel("True") plt.xlabel("Predicted") plt.tight_layout() cm_path = f"{name.lower().replace(' ', '_')}_test_cm.png" plt.savefig(cm_path, dpi=200) plt.show() wandb.log({ f"{name}/test_confusion_matrix": wandb.plot.confusion_matrix( y_true=all_labels, preds=all_preds, class_names=CLASS_NAMES ), f"{name}/test_roc_curve": wandb.plot.roc_curve( y_true=all_labels, y_probas=all_probs, labels=CLASS_NAMES ), f"{name}/test_accuracy": acc, f"{name}/test_precision": prec, f"{name}/test_recall": rec, f"{name}/test_f1": f1, }) return { "model": name, "test_accuracy": acc, "test_precision": prec, "test_recall": rec, "test_f1": f1, "all_preds": all_preds, "all_labels": all_labels, "all_probs": all_probs } # ---- Run Evaluation ---- student_summary = evaluate_model( student, "Student_BioHCL_SSSM_BioPrior_Ablation", test_loader, model_type="biohclsssm" ) # ---- Save + upload evaluation summary CSV ---- summary_df = pd.DataFrame([{ "model": student_summary["model"], "test_accuracy": student_summary["test_accuracy"], "test_precision": student_summary["test_precision"], "test_recall": student_summary["test_recall"], "test_f1": student_summary["test_f1"] }]) summary_df.to_csv("evaluation_summary_ablation_bioprior.csv", index=False) print("\n", summary_df) upload_to_hf("evaluation_summary_ablation_bioprior.csv") # ============================================================ # ROC / PRECISION-RECALL CURVE # ============================================================ device = DEVICE def get_predictions(model, test_loader, is_biohcl=False): model.eval() all_probs, all_targets = [], [] with torch.no_grad(): for images, labels in test_loader: images = images.to(device) outputs = model(images) logits = outputs[0] if is_biohcl else outputs probs = F.softmax(logits, dim=1)[:, 1].cpu().numpy() all_probs.extend(probs) all_targets.extend(labels.numpy()) return np.array(all_targets), np.array(all_probs) print("Running test set inference...") y_true, student_probs = get_predictions(student, test_loader, is_biohcl=True) # ROC Curve fpr_stu, tpr_stu, _ = roc_curve(y_true, student_probs) roc_auc_stu = auc(fpr_stu, tpr_stu) # Precision-Recall Curve prec_stu, rec_stu, _ = precision_recall_curve(y_true, student_probs) pr_auc_stu = average_precision_score(y_true, student_probs) fig, axes = plt.subplots(1, 2, figsize=(14, 6)) # ROC axes[0].plot(fpr_stu, tpr_stu, color='#1f77b4', lw=2.5, label=f'Student BioHCL-SSSM (AUC = {roc_auc_stu:.4f})') axes[0].plot([0, 1], [0, 1], color='gray', lw=1.5, linestyle=':', label='Random Chance') axes[0].set_xlim([0.0, 1.0]); axes[0].set_ylim([0.0, 1.05]) axes[0].set_xlabel('False Positive Rate', fontsize=11) axes[0].set_ylabel('True Positive Rate', fontsize=11) axes[0].set_title('Receiver Operating Characteristic (ROC)', fontsize=13, fontweight='bold') axes[0].legend(loc="lower right", frameon=True) axes[0].grid(True, linestyle=':', alpha=0.6) # PR axes[1].plot(rec_stu, prec_stu, color='#2ca02c', lw=2.5, label=f'Student BioHCL-SSSM (AP = {pr_auc_stu:.4f})') axes[1].set_xlim([0.0, 1.0]); axes[1].set_ylim([0.0, 1.05]) axes[1].set_xlabel('Recall', fontsize=11) axes[1].set_ylabel('Precision', fontsize=11) axes[1].set_title('Precision-Recall (PR) Curve', fontsize=13, fontweight='bold') axes[1].legend(loc="lower left", frameon=True) axes[1].grid(True, linestyle=':', alpha=0.6) plt.tight_layout() roc_pr_path = 'roc_pr_comparison_ablation_bioprior.png' plt.savefig(roc_pr_path, dpi=300) plt.show() print(f"BioHCL-SSSM (BioPrior Ablation) -> ROC-AUC: {roc_auc_stu:.4f} | PR-AUC: {pr_auc_stu:.4f}") # ============================================================ # XAI: Grad-CAM, Attention Map, SHAP # ============================================================ class ModelWrapper(nn.Module): def __init__(self, base_model): super().__init__() self.base_model = base_model def forward(self, x): out = self.base_model(x) return out[0] if isinstance(out, tuple) else out wrapped_student = ModelWrapper(student).to(device) wrapped_student.eval() # Get a single test sample images, labels = next(iter(test_loader)) input_image = images[0:1].to(device) true_label = CLASS_NAMES[labels[0].item()] def unnormalize(img_tensor): img = img_tensor.squeeze().cpu().numpy() if img.ndim == 3: img = np.transpose(img, (1, 2, 0)) img = (img - img.min()) / (img.max() - img.min() + 1e-8) return img orig_img = unnormalize(input_image) # --------------------------------------------------------- # 1. Grad-CAM # --------------------------------------------------------- gradients, activations = [], [] def backward_hook(module, grad_input, grad_output): gradients.append(grad_output[0]) def forward_hook(module, input, output): activations.append(output) target_layer = None for name, module in student.named_modules(): if isinstance(module, nn.Conv2d): target_layer = module handle_fwd = target_layer.register_forward_hook(forward_hook) handle_bwd = target_layer.register_full_backward_hook(backward_hook) output = wrapped_student(input_image) pred_class = output.argmax(dim=1).item() wrapped_student.zero_grad() output[0, pred_class].backward() grad = gradients[0].cpu().data.numpy()[0] act = activations[0].cpu().data.numpy()[0] weights = np.mean(grad, axis=(1, 2)) cam = np.zeros(act.shape[1:], dtype=np.float32) for i, w in enumerate(weights): cam += w * act[i, :, :] cam = np.maximum(cam, 0) cam = cv2.resize(cam, (input_image.shape[3], input_image.shape[2])) cam = cam - np.min(cam) cam = cam / (np.max(cam) + 1e-8) handle_fwd.remove() handle_bwd.remove() plt.figure(figsize=(6, 6)) plt.imshow(orig_img, cmap='gray' if orig_img.ndim == 2 else None) plt.imshow(cam, cmap='jet', alpha=0.5) plt.title(f"Grad-CAM | BioHCL-SSSM (BioPrior Ablation)\nTrue: {true_label} | Pred: {CLASS_NAMES[pred_class]}", fontsize=12, fontweight='bold') plt.axis('off') plt.tight_layout() gradcam_path = "biohcl_ablation_bioprior_gradcam.png" plt.savefig(gradcam_path, dpi=300) plt.show() # --------------------------------------------------------- # 2. Attention Map # --------------------------------------------------------- attention_maps = [] def attn_hook(module, input, output): attention_maps.append(output.detach()) attn_target_layer = None for name, module in student.named_modules(): if "gate" in name.lower() or "attn" in name.lower() or isinstance(module, nn.Sigmoid): attn_target_layer = module if attn_target_layer is None: attn_target_layer = target_layer handle_attn = attn_target_layer.register_forward_hook(attn_hook) with torch.no_grad(): _ = wrapped_student(input_image) handle_attn.remove() attn_out = attention_maps[0].squeeze().cpu().numpy() if attn_out.ndim == 3: attn_map = np.mean(attn_out, axis=0) elif attn_out.ndim == 1: attn_map = np.outer(attn_out, attn_out) else: attn_map = attn_out attn_map = cv2.resize(attn_map, (input_image.shape[3], input_image.shape[2])) attn_map = (attn_map - attn_map.min()) / (attn_map.max() - attn_map.min() + 1e-8) plt.figure(figsize=(6, 6)) plt.imshow(orig_img, cmap='gray' if orig_img.ndim == 2 else None) plt.imshow(attn_map, cmap='inferno', alpha=0.55) plt.title(f"Attention Map | BioHCL-SSSM (BioPrior Ablation)\nTrue: {true_label} | Pred: {CLASS_NAMES[pred_class]}", fontsize=12, fontweight='bold') plt.axis('off') plt.tight_layout() attn_path = "biohcl_ablation_bioprior_attention_map.png" plt.savefig(attn_path, dpi=300) plt.show() # --------------------------------------------------------- # 3. SHAP Visualization # --------------------------------------------------------- mean_t = torch.tensor(IMAGENET_MEAN, device=device).view(1, 3, 1, 1) std_t = torch.tensor(IMAGENET_STD, device=device).view(1, 3, 1, 1) def f_predict(img_batch_nhwc): imgs = torch.from_numpy(img_batch_nhwc).float().permute(0, 3, 1, 2).to(device) imgs = (imgs - mean_t) / std_t with torch.no_grad(): logits = wrapped_student(imgs) probs = F.softmax(logits, dim=1) return probs.cpu().numpy() img_to_plot = orig_img[np.newaxis, ...].astype(np.float32) masker = shap.maskers.Image("blur(64,64)", img_to_plot[0].shape) explainer = shap.Explainer(f_predict, masker, output_names=CLASS_NAMES) shap_values = explainer( img_to_plot, max_evals=500, batch_size=32, outputs=shap.Explanation.argsort.flip[:2], ) shap.image_plot(shap_values, show=False) fig = plt.gcf() fig.suptitle(f"SHAP Explanations | BioHCL-SSSM (BioPrior Ablation)\nTrue: {true_label} | Pred: {CLASS_NAMES[pred_class]}", fontsize=12, fontweight='bold', y=1.05) shap_path = "biohcl_ablation_bioprior_shap.png" plt.savefig(shap_path, dpi=300, bbox_inches='tight') plt.show() # ============================================================ # UPLOAD ALL GENERATED FILES TO HUGGING FACE # ============================================================ files_to_upload = [ "evaluation_summary_ablation_bioprior.csv", "roc_pr_comparison_ablation_bioprior.png", "student_biohcl_sssm_bioprior_ablation_test_cm.png", "biohcl_ablation_bioprior_gradcam.png", "biohcl_ablation_bioprior_attention_map.png", "biohcl_ablation_bioprior_shap.png" ] print("\n" + "="*60) print("πŸ“€ UPLOADING RESULTS TO HUGGING FACE") print("="*60) for file_path in files_to_upload: if os.path.exists(file_path): try: upload_to_hf(file_path) print(f"βœ… Uploaded: {file_path}") except Exception as e: print(f"❌ Failed to upload {file_path}: {e}") else: print(f"⚠️ File not found: {file_path}") print("\n" + "="*60) print("βœ… ALL RESULTS UPLOADED TO HUGGING FACE!") print(f"πŸ“ Repository: https://huggingface.co/{REPO_ID}") print("="*60) wandb.finish() print("βœ… W&B run finished.") print(f"βœ… Checkpoints + evaluation CSV pushed to: https://huggingface.co/{HF_REPO_ID}")
!pip install -q kagglehub wandb huggingface_hub timm albumentations torchvision scikit-learn seaborn
# ============ HARD-CODE YOUR TOKENS / CONFIG HERE ============
HF_TOKEN = os.environ.get('HF_TOKEN', '') # <-- paste your Hugging Face WRITE token
WANDB_API_KEY = "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" # <-- paste your W&B API key (leave "" to get an interactive login prompt)
HF_USERNAME = "unknownBrain" # <-- change this
HF_REPO_ID = f"{HF_USERNAME}/chest-xray-pneumonia-sssm-ablation-bioprior" # HF model repo that will be created/used
# ================================================================
import os
from huggingface_hub import login as hf_login, HfApi
os.environ["HF_TOKEN"] = HF_TOKEN
hf_login(token=HF_TOKEN)
hf_api = HfApi()
print("βœ… Logged in to Hugging Face Hub")
import wandb
if WANDB_API_KEY:
wandb.login(key=WANDB_API_KEY)
else:
wandb.login()
print("βœ… Logged in to Weights & Biases")
import os, glob, random, math
import numpy as np
import pandas as pd
import cv2
from PIL import Image
from tqdm import tqdm
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.optim.lr_scheduler import CosineAnnealingLR
from torch.utils.data import DataLoader, Dataset, WeightedRandomSampler
from sklearn.model_selection import train_test_split
from sklearn.metrics import (classification_report, confusion_matrix, roc_curve, auc,
precision_recall_fscore_support, accuracy_score)
import matplotlib.pyplot as plt
import seaborn as sns
import timm
import albumentations as A
from albumentations.pytorch import ToTensorV2
def get_safe_device():
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
DEVICE = get_safe_device()
print(f"βœ… Training on: {DEVICE}")
SEED = 42
random.seed(SEED); np.random.seed(SEED)
torch.manual_seed(SEED); torch.cuda.manual_seed_all(SEED)
import kagglehub
DATA_DIR = kagglehub.dataset_download("paultimothymooney/chest-xray-pneumonia")
print("Dataset downloaded to:", DATA_DIR)
for root, dirs, files in os.walk(DATA_DIR):
depth = root[len(DATA_DIR):].count(os.sep)
if depth <= 2:
print(root)
CLASS_NAMES = ["NORMAL", "PNEUMONIA"]
NUM_CLASSES = len(CLASS_NAMES)
LABEL_MAP = {name: idx for idx, name in enumerate(CLASS_NAMES)}
def find_chest_xray_root(base_dir):
"""kagglehub extracts a nested folder; locate the level containing train/val/test."""
for root, dirs, files in os.walk(base_dir):
if "train" in dirs and "test" in dirs:
return root
return base_dir
ROOT_DIR = find_chest_xray_root(DATA_DIR)
print("Dataset root:", ROOT_DIR)
def gather_split(split_name):
rows = []
for cls in CLASS_NAMES:
folder = os.path.join(ROOT_DIR, split_name, cls)
if not os.path.isdir(folder):
continue
paths = (glob.glob(os.path.join(folder, "*.jpeg")) +
glob.glob(os.path.join(folder, "*.jpg")) +
glob.glob(os.path.join(folder, "*.png")))
for p in paths:
rows.append({"path": p, "label": cls})
return pd.DataFrame(rows)
df_train_raw = gather_split("train")
df_val_raw = gather_split("val") # tiny/broken split in the original dataset
df_test = gather_split("test") # kept separate, untouched