Download CD-DCU-Net(Incorporate CD-module)/dataloader.py from Li-zhangsheng/CD-DCU-Net: direct link, hf CLI and curl.
- Browser
- Download file 8.41 kB
-
https://huggingface.co/Li-zhangsheng/CD-DCU-Net/resolve/main/CD-DCU-Net(Incorporate%20CD-module)/dataloader.py
- Command line
-
hf download 'hf://Li-zhangsheng/CD-DCU-Net/CD-DCU-Net(Incorporate CD-module)/dataloader.py'
-
curl -L -o dataloader.py 'https://huggingface.co/Li-zhangsheng/CD-DCU-Net/resolve/main/CD-DCU-Net(Incorporate%20CD-module)/dataloader.py'
8.41 kB
| import os | |
| import random | |
| import numpy as np | |
| from PIL import Image | |
| from scipy import io | |
| import torch | |
| from torch.utils.data import Dataset, DataLoader, Subset | |
| from torch.utils.data.distributed import DistributedSampler | |
| import torchvision.transforms.functional as TF | |
| class CustomDataset(Dataset): | |
| def __init__(self, root_dir, crop_size=None, augment=False): | |
| """ | |
| Initialize the dataset. | |
| Args: | |
| root_dir: Dataset root directory. | |
| crop_size: Crop size as an integer or a (height, width) tuple. | |
| augment: Whether to enable data augmentation for the training set. | |
| """ | |
| self.root_dir = root_dir | |
| self.input_dir = os.path.join(root_dir, 'input') | |
| self.output_dir = os.path.join(root_dir, 'output') | |
| # Sort strictly for deterministic multi-GPU and repeated runs. | |
| self.samples = sorted(os.listdir(self.input_dir)) | |
| self.augment = augment | |
| # Process the crop dimensions. | |
| if crop_size is not None: | |
| if isinstance(crop_size, (int, float)): | |
| self.crop_height = self.crop_width = int(crop_size) | |
| elif isinstance(crop_size, (tuple, list)) and len(crop_size) == 2: | |
| self.crop_height, self.crop_width = crop_size | |
| else: | |
| raise ValueError("crop_size should be an integer or a tuple of (height, width)") | |
| else: | |
| self.crop_height = self.crop_width = None | |
| def __len__(self): | |
| """Return the dataset size.""" | |
| return len(self.samples) | |
| def __getitem__(self, idx): | |
| sample_name = self.samples[idx] | |
| # 1. Read each input image: (H, W) -> [1, H, W]. | |
| img_folder = os.path.join(self.input_dir, sample_name) | |
| img1 = torch.from_numpy(np.array(Image.open(os.path.join(img_folder, 'image1.png')), dtype=np.float32)).unsqueeze(0) | |
| img2 = torch.from_numpy(np.array(Image.open(os.path.join(img_folder, 'image2.png')), dtype=np.float32)).unsqueeze(0) | |
| img3 = torch.from_numpy(np.array(Image.open(os.path.join(img_folder, 'image3.png')), dtype=np.float32)).unsqueeze(0) | |
| # Concatenate three images into [3, H, W] so transforms remain synchronized. | |
| input_tensor = torch.cat([img1, img2, img3], dim=0) | |
| # 2. Read the .mat label and convert it to a [1, H, W] tensor. | |
| label_folder = os.path.join(self.output_dir, sample_name) | |
| mat_data = io.loadmat(os.path.join(label_folder, 'label.mat')) | |
| M_tensor = torch.from_numpy(mat_data['M'].astype(np.float32)).unsqueeze(0) | |
| D_tensor = torch.from_numpy(mat_data['D'].astype(np.float32)).unsqueeze(0) | |
| # 3. Apply synchronized geometric augmentation to training data with 75% probability. | |
| if self.augment and random.random() < 0.75: | |
| # Use small random transforms to favor physical patterns over memorization. | |
| angle = random.uniform(-5.0, 5.0) # Rotation: [-5, 5] degrees. | |
| tx = random.randint(-12, 12) # X translation: [-12, 12]. | |
| ty = random.randint(-12, 12) # Y translation: [-12, 12]. | |
| scale = random.uniform(0.95, 1.05) # Scale: [0.95, 1.05]. | |
| # Apply all geometric transforms in one affine operation with bilinear interpolation. | |
| input_tensor = TF.affine(input_tensor, angle=angle, translate=[tx, ty], scale=scale, shear=0, interpolation=TF.InterpolationMode.BILINEAR) | |
| M_tensor = TF.affine(M_tensor, angle=angle, translate=[tx, ty], scale=scale, shear=0, interpolation=TF.InterpolationMode.BILINEAR) | |
| D_tensor = TF.affine(D_tensor, angle=angle, translate=[tx, ty], scale=scale, shear=0, interpolation=TF.InterpolationMode.BILINEAR) | |
| # 4. Compute sina, cosa, and the mask from the transformed physical images. | |
| img1_t, img2_t, img3_t = input_tensor[0], input_tensor[1], input_tensor[2] | |
| sina = -(img1_t * np.sin(2 * np.pi * 0 / 3) + img2_t * np.sin(2 * np.pi * 1 / 3) + img3_t * np.sin(2 * np.pi * 2 / 3)) | |
| cosa = (img1_t * np.cos(2 * np.pi * 0 / 3) + img2_t * np.cos(2 * np.pi * 1 / 3) + img3_t * np.cos(2 * np.pi * 2 / 3)) | |
| # Create the mask in 2D first to avoid a dimension mismatch: [H, W]. | |
| mask_2d = torch.ones_like(img1_t, dtype=torch.float32) | |
| # Apply Boolean indexing while the mask is 2D. | |
| mask_2d[(sina == 0) & (cosa == 0)] = 0 | |
| # Add the channel dimension after assignment to obtain [1, H, W]. | |
| mask_tensor = mask_2d.unsqueeze(0) | |
| # 5. Center-crop the inputs, labels, and mask together. | |
| if self.crop_height is not None and self.crop_width is not None: | |
| _, H, W = input_tensor.shape | |
| start_h = (H - self.crop_height) // 2 | |
| start_w = (W - self.crop_width) // 2 | |
| input_tensor = input_tensor[:, start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] | |
| M_tensor = M_tensor[:, start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] | |
| D_tensor = D_tensor[:, start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] | |
| mask_tensor = mask_tensor[:, start_h:start_h + self.crop_height, start_w:start_w + self.crop_width] | |
| # 6. Apply consistent value normalization by dividing by 1000. | |
| input_tensor = input_tensor / 1000.0 | |
| M_tensor = M_tensor / 1000.0 | |
| D_tensor = D_tensor / 1000.0 | |
| return input_tensor, {'M': M_tensor, 'D': D_tensor, 'mask': mask_tensor} | |
| def seed_everything(seed=42): | |
| """Set global seeds for reproducible results.""" | |
| random.seed(seed) | |
| os.environ['PYTHONHASHSEED'] = str(seed) | |
| np.random.seed(seed) | |
| torch.manual_seed(seed) | |
| torch.cuda.manual_seed(seed) | |
| torch.cuda.manual_seed_all(seed) | |
| torch.backends.cudnn.deterministic = True | |
| torch.backends.cudnn.benchmark = False | |
| def seed_worker(worker_id): | |
| """Set a distinct seed for each DataLoader worker.""" | |
| worker_seed = torch.initial_seed() % 2**32 | |
| np.random.seed(worker_seed) | |
| random.seed(worker_seed) | |
| def create_data_loaders( | |
| data_dir, | |
| batch_size=4, | |
| train_ratio=0.8, | |
| num_workers=0, | |
| crop_size=None, | |
| seed=42, | |
| distributed=False | |
| ): | |
| """ | |
| Create data loaders with a deterministic split and training-only augmentation. | |
| """ | |
| seed_everything(seed) | |
| # Enable augmentation for training while keeping validation images unchanged. | |
| train_dataset_full = CustomDataset(data_dir, crop_size=crop_size, augment=True) | |
| val_dataset_full = CustomDataset(data_dir, crop_size=crop_size, augment=False) | |
| # Split deterministic indices instead of sharing random_split dataset state. | |
| dataset_size = len(train_dataset_full) | |
| indices = list(range(dataset_size)) | |
| np_rng = np.random.RandomState(seed) | |
| np_rng.shuffle(indices) | |
| train_size = int(train_ratio * dataset_size) | |
| train_indices = indices[:train_size] | |
| val_indices = indices[train_size:] | |
| # Separate training and validation data with Subset. | |
| train_dataset = Subset(train_dataset_full, train_indices) | |
| val_dataset = Subset(val_dataset_full, val_indices) | |
| g = torch.Generator() | |
| g.manual_seed(seed) | |
| if distributed: | |
| train_sampler = DistributedSampler(train_dataset, shuffle=True, seed=seed, drop_last=True) | |
| val_sampler = DistributedSampler(val_dataset, shuffle=False, drop_last=False) | |
| else: | |
| train_sampler = None | |
| val_sampler = None | |
| train_loader = DataLoader( | |
| train_dataset, | |
| batch_size=batch_size, | |
| shuffle=(train_sampler is None), | |
| sampler=train_sampler, | |
| num_workers=num_workers, | |
| pin_memory=True, | |
| drop_last=True, | |
| persistent_workers=(num_workers > 0), | |
| worker_init_fn=seed_worker, | |
| generator=g, | |
| ) | |
| val_loader = DataLoader( | |
| val_dataset, | |
| batch_size=batch_size, | |
| shuffle=False, | |
| sampler=val_sampler, | |
| num_workers=num_workers, | |
| pin_memory=True, | |
| drop_last=False, # Keep every validation sample for complete evaluation. | |
| persistent_workers=(num_workers > 0), | |
| worker_init_fn=seed_worker, | |
| generator=g, | |
| ) | |
| return train_loader, val_loader | |
| if __name__ == "__main__": | |
| # Basic smoke test. | |
| print("Dataloader script is ready!") | |