FEAT: Complete full data preparation pipeline

data_preparation.py

@@ -1,4 +1,5 @@
 # --- Standard Python Library ---
+import os
 import random
 
 # --- Data Handling & Analysis ---
@@ -16,7 +17,7 @@ from torchvision import transforms
 from torch.utils.data import DataLoader
 
 # --- Experiment Tracking ---
-from clearml import Task, Dataset
+from clearml import Task, Logger
 
 
 # Setting up the SEED to be able to repeat experiments
@@ -29,6 +30,7 @@ if torch.cuda.is_available():
 
 
 # Initialising a task on ClearML
+# UPDATE CLEARML
 task = Task.init(project_name= 'smallGroupProject', task_name = 'data_prep')
 task.set_random_seed(SEED)
 clearml_logger = task.get_logger()
@@ -168,7 +170,48 @@ print("✅ Checkpoint: Plot with classes distributions is created and saved")
 
 
 # --------------- Data Splits ------------
+def get_transform_pipelines():
+    """
+    Defines and returns the normalization and augmentation pipelines.
+    """
+    # Standard ImageNet mean and std 
+    # These values are used to normalize the tensors
+    IMAGENET_MEAN = [0.485, 0.456, 0.406]
+    IMAGENET_STD = [0.229, 0.224, 0.225]
+
+    # Defining pipeline to ensure that images are consistently formatted (for Val/Test)
+    normalisation_pipeline = transforms.Compose([
+        # Convert PIL Image to a PyTorch Tensor
+        # This also scales pixel values from [0, 255] to [0.0, 1.0]
+        transforms.ToTensor(), 
+
+        # Normalise the Tensor; Standartises pixel values
+        transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
+    ])
+    print("✅ Checkpoint: Transform pipeline created")
+
+    # Augmentation pipeline (to change some parameters of the pictures to create "new" ones)
+    augmentation_pipeline = transforms.Compose([
+        # Randomly changing some parameters of pictures to enrich dataset
+        transforms.RandomRotation(degrees=30),
+        transforms.ColorJitter(brightness=0.2, saturation=0.2),
+        transforms.GaussianBlur(kernel_size=3),
+
+        # Convert to Tensor and Normalise
+        transforms.ToTensor(), 
+        transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
+    ])
+
+    print("✅ Checkpoint: Augmentation pipeline created")
+    
+    # Return both pipelines
+    return normalisation_pipeline, augmentation_pipeline
+
+
 def get_prototype_loaders(batch_size=32):
+    """
+    Creates and returns DataLoaders for the 25% PROTOTYPE dataset.
+    """
     # Calling function to define pipelines
     normalisation_pipeline, augmentation_pipeline = get_transform_pipelines()
 
@@ -189,7 +232,7 @@ def get_prototype_loaders(batch_size=32):
     proto_val_split = split_2_dict['train']
     proto_test_split = split_2_dict['test']
 
-    print("✅ Checkpoint: Dataset splitted")
+    print("✅ Checkpoint: Prototype Dataset splitted")
 
     # -- Putting splits through pipelines --
     proto_train_split.set_transform(augmentation_pipeline)
@@ -201,16 +244,18 @@ def get_prototype_loaders(batch_size=32):
     proto_val_loader = DataLoader(dataset = proto_val_split, batch_size = batch_size, shuffle = False )
     proto_test_loader = DataLoader(dataset = proto_test_split, batch_size = batch_size, shuffle = False )
 
-    print("✅ Checkpoint: DataLoaders are set")
+    print("✅ Checkpoint: Prototype DataLoaders are set")
     return proto_train_loader, proto_val_loader, proto_test_loader
 
 
-
 def get_final_loaders(batch_size=32):
+    """
+    Creates and returns DataLoaders for the 100% FINAL dataset.
+    """
     # Calling function to define pipelines
     normalisation_pipeline, augmentation_pipeline = get_transform_pipelines()
 
-    # -- Split the prototype dataset -- 
+    # -- Split the FULL dataset -- 
     # This returns a dictionary: {'train': 70%, 'test': 30%}
     split_1_dict = data_plants.train_test_split(test_size=0.3, seed=SEED)
 
@@ -227,49 +272,21 @@ def get_final_loaders(batch_size=32):
     val_split = split_2_dict['train']
     test_split = split_2_dict['test']
 
-    print("✅ Checkpoint: Dataset splitted")
+    print("✅ Checkpoint: Final Dataset splitted")
 
     # -- Putting splits through pipelines --
     train_split.set_transform(augmentation_pipeline)
     val_split.set_transform(normalisation_pipeline)
     test_split.set_transform(normalisation_pipeline)
 
-    # -- Creating the prototype dataloaders -- 
+    # -- Creating the final dataloaders -- 
     train_loader = DataLoader(dataset = train_split, batch_size = batch_size, shuffle = True )
     val_loader = DataLoader(dataset = val_split, batch_size = batch_size, shuffle = False )
     test_loader = DataLoader(dataset = test_split, batch_size = batch_size, shuffle = False )
 
-    print("✅ Checkpoint: DataLoaders are set")
+    print("✅ Checkpoint: Final DataLoaders are set")
     return train_loader, val_loader, test_loader
 
-
-def get_transform_pipelines():
-    """
-    Defines and returns the normalization and augmentation pipelines.
-    """
-    IMAGENET_MEAN = [0.485, 0.456, 0.406]
-    IMAGENET_STD = [0.229, 0.224, 0.225]
-
-    # Defining pipeline for Val/Test
-    normalisation_pipeline = transforms.Compose([
-        transforms.ToTensor(), 
-        transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
-    ])
-
-    # Augmentation pipeline for Train
-    augmentation_pipeline = transforms.Compose([
-        transforms.RandomRotation(degrees=30),
-        transforms.ColorJitter(brightness=0.2, saturation=0.2),
-        transforms.GaussianBlur(kernel_size=3),
-        transforms.ToTensor(), 
-        transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)
-    ])
-    
-    print("✅ Checkpoint: Transform pipelines created")
-    
-    # Return both pipelines
-    return normalisation_pipeline, augmentation_pipeline
-
 # ----------------------------------------------------------------------
 if __name__ == "__main__":