import argparse import os import torch from PIL import Image from torch.utils.data import Dataset, DataLoader, random_split from tqdm import tqdm MODEL_DIR = "./blip-xray-finetuned" CHECKPOINT_PATH = os.path.join(MODEL_DIR, "xray_blip.pth") ONNX_PATH = os.path.join(MODEL_DIR, "onnx") class RadiologyCaptionDataset(Dataset): def __init__(self, hf_dataset, max_samples=None): self.data = [] for i, example in enumerate(hf_dataset): if max_samples and i >= max_samples: break image = example.get("image") caption = example.get("caption", "").strip() if image is None or not caption: continue if image.mode != "RGB": image = image.convert("RGB") self.data.append((image, caption)) def __len__(self): return len(self.data) def __getitem__(self, idx): return self.data[idx] def collate_fn(batch, processor): images = [item[0] for item in batch] captions = [item[1] for item in batch] encoding = processor( images=images, text=captions, return_tensors="pt", padding=True, truncation=True, max_length=128 ) encoding["labels"] = encoding["input_ids"].clone() return encoding def train(epochs, batch_size, lr, max_samples, resume, val_split, use_amp, grad_accum): from transformers import BlipProcessor, BlipForConditionalGeneration from datasets import load_dataset has_gpu = torch.cuda.is_available() use_amp = use_amp and has_gpu scaler = torch.cuda.amp.GradScaler() if use_amp else None hf_ds = load_dataset("eltorio/ROCOv2-radiology", split="train", streaming=True) processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") if resume and os.path.exists(CHECKPOINT_PATH): print(f"Resuming from checkpoint: {CHECKPOINT_PATH}") model = BlipForConditionalGeneration.from_pretrained(MODEL_DIR) start_epoch = torch.load(CHECKPOINT_PATH, weights_only=False).get("epoch", 0) + 1 else: model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") start_epoch = 0 if has_gpu: model = model.cuda() from functools import partial _collate = partial(collate_fn, processor=processor) full_dataset = RadiologyCaptionDataset(hf_ds, max_samples) val_size = max(int(val_split * len(full_dataset)), 1) train_size = len(full_dataset) - val_size train_subset, val_subset = random_split(full_dataset, [train_size, val_size]) train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True, collate_fn=_collate) val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False, collate_fn=_collate) print(f"Dataset: {len(train_subset)} train + {len(val_subset)} val samples") print(f"Device: {'GPU' if has_gpu else 'CPU'} AMP: {'ON' if use_amp else 'OFF'} Grad accum: {grad_accum}") optimizer = torch.optim.AdamW(model.parameters(), lr=lr) model.train() for epoch in range(start_epoch, epochs): print(f"\n{'='*40}") print(f" Epoch {epoch + 1}/{epochs}") print(f"{'='*40}") # ── Train ── total_loss = 0.0 optimizer.zero_grad() pbar = tqdm(train_loader, desc=f" Train") for i, batch in enumerate(pbar): pixel_values = batch.get("pixel_values") input_ids = batch.get("input_ids") attention_mask = batch.get("attention_mask") labels = batch.get("labels") if has_gpu: pixel_values = pixel_values.cuda() input_ids = input_ids.cuda() attention_mask = attention_mask.cuda() labels = labels.cuda() with torch.amp.autocast("cuda", enabled=use_amp): outputs = model( pixel_values=pixel_values, input_ids=input_ids, attention_mask=attention_mask, labels=labels, ) loss = outputs.loss / grad_accum if use_amp: scaler.scale(loss).backward() else: loss.backward() if (i + 1) % grad_accum == 0 or (i + 1) == len(train_loader): if use_amp: scaler.step(optimizer) scaler.update() else: optimizer.step() optimizer.zero_grad() total_loss += loss.item() * grad_accum pbar.set_postfix(loss=f"{loss.item() * grad_accum:.4f}") avg_train_loss = total_loss / len(train_loader) # ── Validation ── model.eval() val_loss = 0.0 with torch.no_grad(): for batch in tqdm(val_loader, desc=f" Val"): pixel_values = batch.get("pixel_values") input_ids = batch.get("input_ids") attention_mask = batch.get("attention_mask") labels = batch.get("labels") outputs = model( pixel_values=pixel_values, input_ids=input_ids, attention_mask=attention_mask, labels=labels, ) val_loss += outputs.loss.item() model.train() avg_val_loss = val_loss / len(val_loader) print(f" Train loss: {avg_train_loss:.4f} | Val loss: {avg_val_loss:.4f}") os.makedirs(MODEL_DIR, exist_ok=True) model.save_pretrained(MODEL_DIR) processor.save_pretrained(MODEL_DIR) torch.save({"epoch": epoch}, CHECKPOINT_PATH) print(f" Checkpoint saved to {MODEL_DIR}") print(f"\nTraining complete. Model saved to {MODEL_DIR}") def evaluate(batch_size, max_samples): from transformers import BlipProcessor, BlipForConditionalGeneration from datasets import load_dataset if not os.path.exists(MODEL_DIR): print(f"No model found at {MODEL_DIR}. Run training first.") return from functools import partial print("Loading model and dataset...") processor = BlipProcessor.from_pretrained(MODEL_DIR) model = BlipForConditionalGeneration.from_pretrained(MODEL_DIR) model.eval() _collate = partial(collate_fn, processor=processor) hf_ds = load_dataset("eltorio/ROCOv2-radiology", split="train", streaming=True) dataset = RadiologyCaptionDataset(hf_ds, max_samples) loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, collate_fn=_collate) try: from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction smoothie = SmoothingFunction().method4 except ImportError: print("nltk not installed. Skipping BLEU evaluation.") print("Install with: pip install nltk") return print(f"Evaluating on {len(dataset)} samples...") references = [] hypotheses = [] total_loss = 0.0 with torch.no_grad(): for batch in tqdm(loader, desc="Evaluating"): pixel_values = batch.get("pixel_values") input_ids = batch.get("input_ids") attention_mask = batch.get("attention_mask") labels = batch.get("labels") outputs = model( pixel_values=pixel_values, input_ids=input_ids, attention_mask=attention_mask, labels=labels, ) total_loss += outputs.loss.item() generated_ids = model.generate(pixel_values=pixel_values, max_length=64) for i in range(len(labels)): ref = processor.decode(labels[i], skip_special_tokens=True) hyp = processor.decode(generated_ids[i], skip_special_tokens=True) if ref and hyp: references.append([ref.split()]) hypotheses.append(hyp.split()) avg_loss = total_loss / len(loader) bleu = corpus_bleu(references, hypotheses, smoothing_function=smoothie) print(f"Average loss: {avg_loss:.4f}") print(f"Corpus BLEU: {bleu:.4f}") print("\nSample generations:") for i in range(min(5, len(references))): ref = " ".join(references[i][0]) hyp = " ".join(hypotheses[i]) print(f" REF: {ref[:120]}") print(f" HYP: {hyp[:120]}") print() def export_onnx(): print("[yellow]BLIP ONNX export requires a newer version of optimum.[/yellow]") print("[yellow]Run: pip install --upgrade optimum[/yellow]") print("[yellow]Until then, the system uses PyTorch directly (no speed difference for inference).[/yellow]") def generate(image_path): from transformers import BlipProcessor, BlipForConditionalGeneration if not os.path.exists(MODEL_DIR): print(f"No model found at {MODEL_DIR}. Run training first.") return print(f"Loading model from {MODEL_DIR}...") processor = BlipProcessor.from_pretrained(MODEL_DIR) model = BlipForConditionalGeneration.from_pretrained(MODEL_DIR) model.eval() image = Image.open(image_path).convert("RGB") inputs = processor(images=image, return_tensors="pt") with torch.no_grad(): out = model.generate(**inputs, max_length=64) caption = processor.decode(out[0], skip_special_tokens=True) print(f"Generated caption: {caption}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Fine-tune BLIP for radiology caption generation") parser.add_argument("--mode", required=True, choices=["train", "evaluate", "export-onnx", "generate"]) parser.add_argument("--epochs", type=int, default=3) parser.add_argument("--batch_size", type=int, default=4) parser.add_argument("--lr", type=float, default=5e-5) parser.add_argument("--max_samples", type=int, default=500, help="Max samples for training/eval (remove to use all)") parser.add_argument("--resume", action="store_true", help="Resume from last checkpoint") parser.add_argument("--val_split", type=float, default=0.1, help="Fraction of data for validation") parser.add_argument("--use_amp", action="store_true", help="Enable mixed precision (GPU only)") parser.add_argument("--grad_accum", type=int, default=1, help="Gradient accumulation steps") parser.add_argument("--image", help="Path to image for --mode generate") args = parser.parse_args() if args.mode == "train": train(args.epochs, args.batch_size, args.lr, args.max_samples, args.resume, args.val_split, args.use_amp, args.grad_accum) elif args.mode == "evaluate": evaluate(args.batch_size, args.max_samples) elif args.mode == "export-onnx": export_onnx() elif args.mode == "generate": if not args.image: print("--mode generate requires --image ") else: generate(args.image)