Merge branch 'develop' of https://github.kcl.ac.uk/K23064919/smallGroupProject into develop

dataPrep/data_preparation.py

@@ -74,15 +74,15 @@ task.connect({
 })
 
 # ----- Load a subset from a given dataset & track with ClearML -----
-data_plants, prototyping_dataset, features, clearml_dataset = make_subset(
-    DATASET_LINK, DATASET_SUBSET_RATIO, clearml_logger
+data_plants, subset_dataset, features = make_subset(
+    DATASET_LINK, DATASET_SUBSET_RATIO, task
 )
 
 
 # ---- Exploratory data analysis (EDA) ----
 
 # Reformatting the label feature to understand bias
-labels_list = prototyping_dataset['label']
+labels_list = subset_dataset['label']
 df_labels = pd.Series(labels_list)
 label_count = df_labels.value_counts(sort=False)
 
@@ -111,6 +111,7 @@ clearml_logger.report_scalar(
     value=(max_count / min_count), 
     iteration=1
 )
+
 print("--- Class imbalance analysis --- ")
 print(f"Max labels in a class: {max_count}")
 print(f"Min labels in a class: {min_count}")
@@ -122,16 +123,17 @@ class_names = features['label'].names
 formatted_class_names = [" ".join(name.replace('_', ' ').split()) for name in class_names]
 label_count.index = formatted_class_names
 
+# Plotting class distribution
 plt.figure(figsize=(10,6))
 label_count.plot(kind='bar', color='skyblue')
-plt.title("Class Distribution in Prototype Dataset")
+plt.title("Class Distribution in Subset Dataset")
 plt.xlabel("Class")
 plt.ylabel("Count")
 plt.tight_layout()
 
 clearml_logger.report_matplotlib_figure(
     title="EDA Class Distribution",
-    series="Prototype Subset",
+    series="Subset Dataset",
     figure=plt.gcf(),
     iteration=1
 )
@@ -149,7 +151,7 @@ if __name__ == "__main__":
     }
 
     prototype_loaders = make_dataset_loaders(
-        prototyping_dataset, SEED, BATCH_SIZE, TEST_SIZE, aug_config
+        subset_dataset, SEED, BATCH_SIZE, TEST_SIZE, aug_config
     )
 
     print("\n--- Handoff Test Successful ---")
@@ -173,14 +175,9 @@ if __name__ == "__main__":
     print(f"Validation loader batches: {len(final_loaders['val'])}")
     print(f"Test loader batches: {len(final_loaders['test'])}")
 
-    # Record dataset info in ClearML
-    task.connect_configuration(
-        {"dataset_id": clearml_dataset.id},
-        name="Dataset Metadata"
-    )
-    task.mark_completed()
 
-    
     # Close the ClearML task 
+    task.mark_completed()
     task.close()
+
     print("\n--- Script Finished ---")

dataPrep/helpers/clearml_data.py

@@ -0,0 +1,136 @@
+import os
+import numpy as np
+
+from clearml import Task, Dataset
+from datasets import load_dataset
+from dataPrep.helpers.transforms_loaders import make_dataset_loaders
+
+
+'''
+Takes latest Data Prep ClearML task from project and reconstruct:
+- data loaders for both full and subset datasets
+- Aug settings used
+'''
+def extract_latest_data_task(project_name: str = "Small Group Project"):
+
+  # --------- Get latest Data Preparation task from ClearML ---------
+
+  all_tasks = Task.get_tasks(
+    project_name=project_name,
+    allow_archived=False,
+    task_filter={'order_by': ["-last_update"]},
+  )
+
+  if not all_tasks:
+      raise RuntimeError(f"No tasks found in project '{project_name}'")
+
+  dp_tasks = [
+    t for t in all_tasks
+    if t.task_type == Task.TaskTypes.data_processing
+    and t.completed is not None
+  ]
+
+  if not dp_tasks:
+      raise RuntimeError("No 'Data Preparation' tasks found in this project!")
+
+  # Latest Data Prep Task 
+  latest_task = dp_tasks[0]
+  DYNAMIC_TASK_ID = latest_task.id
+  DATA_PREP = Task.get_task(task_id=DYNAMIC_TASK_ID)
+
+  # Load subset indices artifact from Data Prep task
+  artifacts = DATA_PREP.artifacts
+  if "subset_indices" not in artifacts:
+      raise RuntimeError("Data Prep task did not upload 'subset_indices' artifact!")
+
+  artifact = artifacts["subset_indices"]
+  subset_indices_path = artifact.get_local_copy()
+  subset_indices = np.load(subset_indices_path)
+
+  # Load dataset metadata from Data Prep task
+  data_params = DATA_PREP.get_parameters()
+
+  subset_ratio = float(data_params['General/dataset/subset_ratio'])
+  dataset_link = data_params['General/dataset/link']
+  seed = int(data_params['General/seed'])
+  batch_size = int(data_params['General/dataloaders/batch_size'])
+  test_size = float(data_params['General/dataloaders/test_size'])
+
+  aug_config = {
+      'rotation': float(data_params['General/augmentation/rotation']),
+      'brightness': float(data_params['General/augmentation/brightness']),
+      'saturation': float(data_params['General/augmentation/saturation']),
+      'blur': float(data_params['General/augmentation/blur']),
+  }
+
+
+  # Load Full Dataset 
+  try:
+      ds = load_dataset(dataset_link)
+  except Exception as e:
+      raise RuntimeError(f"Error loading the dataset: {e}")
+
+  full_dataset = ds['train']
+
+  # Apply subset indices to full dataset - this gives you the same subset as data prep
+  subset_dataset = full_dataset.select(subset_indices)
+
+  # Get data loaders for both full and subset datasets
+  subset_loaders, full_loaders, aug_config = get_data_loaders(data_params, subset_dataset, full_dataset)
+  batch_size = int(data_params['General/dataloaders/batch_size'])
+  seed = int(data_params['General/seed'])
+
+
+  # Gather data prep task metadata
+  data_prep_metadata = {
+    "data_prep_task_id": DYNAMIC_TASK_ID,
+    "dataset_link": dataset_link,
+    "subset_ratio_used": subset_ratio,
+    "augmentation_used": aug_config,
+    "batch_size_used": batch_size,
+    "seed_used": seed,
+    "test_size_used": test_size
+  }
+
+  return subset_loaders, full_loaders, data_prep_metadata
+
+
+'''
+Takes a given dataset, subset, data params to create DataLoaders
+Loaders split data into train, val, test
+'''
+def get_data_loaders(data_params, subset_dataset, full_dataset):
+
+  # Extract data parameters- these will be used in the DataLoaders
+  seed = int(data_params['General/seed'])
+  batch_size = int(data_params['General/dataloaders/batch_size'])
+  test_size = float(data_params['General/dataloaders/test_size'])
+
+  aug_config = {
+      'rotation': float(data_params['General/augmentation/rotation']),
+      'brightness': float(data_params['General/augmentation/brightness']),
+      'saturation': float(data_params['General/augmentation/saturation']),
+      'blur': float(data_params['General/augmentation/blur'])
+  }
+
+  # Create DataLoaders using the parameters from data prep
+  subset_loaders = make_dataset_loaders(
+      subset_dataset, seed, batch_size, test_size, aug_config
+  )
+
+  print("\n--- Handoff Test Successful ---")
+  print(f"Prototype Train loader batches: {len(subset_loaders['train'])}")
+  print(f"Prototype Validation loader batches: {len(subset_loaders['val'])}")
+  print(f"Prototype Test loader batches: {len(subset_loaders['test'])}")
+
+
+  full_loaders = make_dataset_loaders(
+      full_dataset, seed, batch_size, test_size, aug_config
+  )
+
+  print("\n--- Handoff Test Successful ---")
+  print(f"Train loader batches: {len(full_loaders['train'])}")
+  print(f"Validation loader batches: {len(full_loaders['val'])}")
+  print(f"Test loader batches: {len(full_loaders['test'])}")
+
+  return subset_loaders, full_loaders, aug_config

dataPrep/helpers/create_dataset.py

@@ -6,7 +6,6 @@ import os
 import random
 import numpy as np
 from datasets import load_dataset
-from clearml import Dataset
 
 
 '''
@@ -14,7 +13,7 @@ Load a DS from HuggingFace Link & randomly subset it - upload subset to ClearML
 Subset indicies are uploaded to ClearML for reproducibility
 REPRODUCE: Load full DS, then load indicies from ClearML to get same subset
 '''
-def make_subset(dataset_link, subset_ratio, clearml_logger):
+def make_subset(dataset_link, subset_ratio, clearml_task):
 
     # Load dataset
     try:
@@ -34,36 +33,16 @@ def make_subset(dataset_link, subset_ratio, clearml_logger):
     random.shuffle(indices)
     subset_indices = indices[:subset_size]
 
-    prototyping_dataset = data_plants.select(subset_indices)
-# I THINK WE NEED TO REMOVE THIS LATER
-# We dont really need to upload subset everytime (Im not sure tho)
-    # Register subset in ClearML 
-    clearml_dataset = Dataset.create(
-        dataset_name="Plant Village Prototype",
-        dataset_project="Small Group Project",
-        dataset_tags=["prototype", "subset"],
-        use_current_task=False
-    )
-    clearml_dataset.add_tags([
-        f"subset_ratio_{subset_ratio}",
-        "hf_source"
-    ])
+    subset_dataset = data_plants.select(subset_indices)
 
-    # Save indices
+    # -------- Upload the subset indices as a ClearML artifact --------
     subset_path = "subset_indices.npy"
     np.save(subset_path, subset_indices)
-    clearml_dataset.add_files(subset_path)
-    clearml_dataset.set_metadata({
-        "huggingface_dataset": dataset_link,
-        "subset_ratio": subset_ratio,
-        "total_samples": len(prototyping_dataset)
-    })
-
-    clearml_dataset.upload()
-    clearml_dataset.finalize()
-    clearml_logger.report_text(f"Created ClearML Dataset: {clearml_dataset.id}")
 
-    # Clean up local file
-    os.remove(subset_path)
+    clearml_task.upload_artifact(
+        name="subset_indices",
+        artifact_object=subset_path
+    )
+    clearml_task.get_logger().report_text(f"Uploaded subset indices as artifact: {subset_path}")
 
-    return data_plants, prototyping_dataset, features, clearml_dataset
+    return data_plants, subset_dataset, features

requirements.txt

@@ -1,29 +1,19 @@
 
 # Core dependencies
-torch>=2.0.0
-torchvision>=0.15.0
-gradio>=4.0.0
-numpy>=1.24.0
-Pillow>=10.0.0
+torch==2.2.2
+torchvision==0.17.2
+torcheval==0.0.7
+numpy==1.26.4
+Pillow==10.3.0
+gradio==4.19.0
 
-# For model deployment and tracking
-huggingface-hub>=0.19.0
-clearml>=1.14.0
-
-# Optional: for advanced features
-datasets>=2.14.0  # For loading PlantVillage dataset from HuggingFace
-# -- Data prep requirements --
 # Data Handling & Analysis
-numpy
-pandas
-datasets
+pandas==2.2.2
+datasets==2.18.0
 
 # Visualization
-matplotlib
+matplotlib==3.8.4
 
-# PyTorch (Machine Learning)
-torch
-torchvision
-
-# Experiment Tracking
-clearml
+# For model deployment and tracking
+huggingface-hub==0.23.0
+clearml==2.0.2

trainingModel/Training.py

@@ -15,10 +15,10 @@ def train_model(
     device: torch.device,
     n_epochs: int = 4,
     lr: float = 1e-3,
+    num_classes: int = 39,
+    optimizer_type: str = "adam",
+    flatten_input: bool = False,
     save_path: str = "best_model.pt",
-    flatten_input = False,
-    num_classes : int = 39,
-
 ):
     """
     Trains the given model and returns:
@@ -40,7 +40,11 @@ def train_model(
 
     # Loss and optimizer
     criterion = nn.CrossEntropyLoss()
-    optimizer = torch.optim.Adam(model.parameters(), lr=lr ) # might add momentum 0.9 later
+
+    if optimizer_type.lower() == "adam":
+        optimizer = torch.optim.Adam(model.parameters(), lr=lr ) # might add momentum 0.9 later
+    else:
+        optimizer = torch.optim.AdamW(model.parameters(), lr=lr )
 
     # Metric trackers 
     train_accuracy_fn = MulticlassAccuracy(num_classes=num_classes)
@@ -49,20 +53,31 @@ def train_model(
     # Arrays to log metrics
     num_batches = len(train_loader)
 
+    # Batch-level logs
+    batch_losses = []
+    batch_accuracies = []
+
+    # Epoch-level logs
+    epoch_losses = np.zeros(n_epochs)
+    epoch_accuracies = np.zeros(n_epochs)
+    val_accuracies = np.zeros(n_epochs)
+
+
     if num_batches == 0:
         raise RuntimeError("UH OH!!!! empty train loader")
 
     # Store training losses and accuracies for every batch
     # num_batches is the number of batches for every epoch
-    training_losses = np.zeros(num_batches * n_epochs)
-    training_accuracies = np.zeros(num_batches * n_epochs)
+    #training_losses = np.zeros(num_batches * n_epochs)
+    #training_accuracies = np.zeros(num_batches * n_epochs)
 
     # store validation accuracy for every epoch
-    val_accuracies = np.zeros(n_epochs)
+
 
     # keep track of best validation accuracy and best model
     best_accuracy = 0.0
 
+
     #----------------------
     # training loop
     #----------------------
@@ -71,8 +86,12 @@ def train_model(
         model.train()
         train_accuracy_fn.reset()
 
+        running_loss = 0.0
+        running_correct = 0
+        running_total = 0
+
         # iterate over all the dataloader's mini-batches
-        for i, batch in enumerate(train_loader):
+        for batch in train_loader:
 
             # move to GPU memory
             inputs = batch["image"].to(device)
@@ -88,22 +107,30 @@ def train_model(
             outputs = model(inputs)
             loss = criterion(outputs, labels)
             
-            # Backward pass
+            # Backward pass & update params
             loss.backward()
-
-            # updates the parameters
             optimizer.step()
-            
-            # log the loss value
-            training_losses[epoch * num_batches + i] = loss.item() 
 
-            #updates the accuracy computation with new data 
-            train_accuracy_fn.update(outputs, labels)
+            # Log batch-level metrics
+            batch_losses.append(loss.item())
+            batch_acc = (outputs.argmax(dim=1) == labels).float().mean().item()
+            batch_accuracies.append(batch_acc)
+
+            # Sum epoch stats
+            running_loss += loss.item() * inputs.size(0)
+            running_correct += (outputs.argmax(dim=1) == labels).sum().item()
+            running_total += labels.size(0)
+
 
-            #compute accuracy with the current data
-            training_accuracies[epoch * num_batches + i] = train_accuracy_fn.compute().item() 
+        # Epoch-level metrics (average over all batches)
+        epoch_loss_avg = running_loss / running_total
+        epoch_acc_avg = running_correct / running_total
 
-        print(f'Epoch {epoch + 1} training complete')
+        epoch_losses[epoch] = epoch_loss_avg
+        epoch_accuracies[epoch] = epoch_acc_avg
+
+        print(f"\n--- Epoch {epoch + 1}: ---")
+        print(f'Train loss={epoch_loss_avg:.4f}\nTrain accuracy={epoch_acc_avg:.4f}\n')
 
         # ----------------------
         # validation loop
@@ -123,25 +150,30 @@ def train_model(
                     inputs = inputs.view(inputs.size(0), -1)
 
                 outputs = model(inputs)
-
                 val_accuracy_fn.update(outputs, labels)
 
-        current_accuracy = val_accuracy_fn.compute().item()
-        val_accuracies[epoch] = current_accuracy
+        
+        current_val_accuracy = val_accuracy_fn.compute().item()
+        val_accuracies[epoch] = current_val_accuracy
+
+        print(f"\nEpoch {epoch+1}: val acc={current_val_accuracy:.4f}")
 
         # keep track of best validation accuracy and save best model so far
-        if current_accuracy > best_accuracy:
-            best_accuracy = current_accuracy
+        if current_val_accuracy > best_accuracy:
+            best_accuracy = current_val_accuracy
             torch.save(model.state_dict(), save_path)
-            print(f'Epoch {epoch + 1} (validation accuracy: {best_accuracy})')
-        print(f'Epoch {epoch + 1} validation complete')
+        
+
+        print(f'Epoch {epoch + 1} validation complete\n')
 
     print(f"\nTraining finished. Best val accuracy: {best_accuracy:.4f}")
     print(f"Best model weights saved to: {save_path}")
 
     training_metrics = {
-        "losses": training_losses,
-        "accuracies": training_accuracies,
+        "batch_losses": np.array(batch_losses),
+        "batch_accuracies": np.array(batch_accuracies),
+        "epoch_losses": epoch_losses,
+        "epoch_accuracies": epoch_accuracies,
         "val_accuracies": val_accuracies,
         "best_accuracy": best_accuracy,
     }

trainingModel/run_training.py

@@ -1,105 +1,15 @@
-import os
-import numpy as np
-from clearml import Task, Dataset
-from datasets import load_dataset
 
-# Latest Data Prep Task 
-all_tasks = Task.get_tasks(project_name="Small Group Project")
-if not all_tasks:
-    raise RuntimeError("No tasks found in project 'Small Group Project'")
+from clearml import Task
+from dataPrep.helpers.clearml_data import extract_latest_data_task
 
-dp_tasks = [t for t in all_tasks if t.name == "Data Preparation"]
-if not dp_tasks:
-    raise RuntimeError("No 'Data Preparation' tasks found in this project!")
+import torch
+from models.modelOne import modelOne
+from trainingModel.Training import train_model
 
-<<<<<<< HEAD
-# -------------- Load Data --------------
-
-all_tasks = Task.get_tasks(project_name="Small Group Project")
-if not all_tasks:
-    raise RuntimeError("No tasks found in project 'Small Group Project'")
-
-dp_tasks = [t for t in all_tasks if t.name == "Data Preparation"]
-if not dp_tasks:
-    raise RuntimeError("No 'Data Preparation' tasks found in this project!")
-
-# Latest Data Prep Task 
-latest_task = max(dp_tasks, key=lambda t: t.id)
-DYNAMIC_TASK_ID = latest_task.id
-DATA_PREP = Task.get_task(task_id=DYNAMIC_TASK_ID)
-
-=======
-latest_task = max(dp_tasks, key=lambda t: t.id)
-DYNAMIC_TASK_ID = latest_task.id
-DATA_PREP = Task.get_task(task_id=DYNAMIC_TASK_ID)
-
->>>>>>> 20050ad82ebca27a376e15837a7abf79fca23e98
-# Dataset ID 
-config_objects = DATA_PREP.get_configuration_objects()
-raw_meta = config_objects["Dataset Metadata"]
-dataset_id = raw_meta.split("=")[1].strip().replace('"', "")
-
-# Load ClearML Dataset
-subset_clearml = Dataset.get(dataset_id=dataset_id)
-local_folder = subset_clearml.get_local_copy()
-
-<<<<<<< HEAD
-subset_indices = np.load(os.path.join(local_folder, "subset_indices.npy"))
-=======
-subset_indices_path = os.path.join(local_folder, "subset_indices.npy")
-subset_indices = np.load(subset_indices_path)
->>>>>>> 20050ad82ebca27a376e15837a7abf79fca23e98
-
-# Load Dataset Parameters 
-data_params = DATA_PREP.get_parameters()
-dataset_link = data_params['General/dataset/link']
-
-# Load Full Dataset 
-try:
-    ds = load_dataset(dataset_link)
-except Exception as e:
-    raise RuntimeError(f"Error loading the dataset: {e}")
-
-full_dataset = ds['train']
-
-
-
-# Apply subset indices to full dataset - this gives you the same subset as data prep
-subset_dataset = full_dataset.select(subset_indices)
 
-
-# Extract parameters from data prep task - these will create the DataLoaders
-seed = int(data_params['General/seed'])
-batch_size = int(data_params['General/dataloaders/batch_size'])
-test_size = float(data_params['General/dataloaders/test_size'])
-
-aug_config = {
-    'rotation': float(data_params['General/augmentation/rotation']),
-    'brightness': float(data_params['General/augmentation/brightness']),
-    'saturation': float(data_params['General/augmentation/saturation']),
-    'blur': float(data_params['General/augmentation/blur'])
-}
-
-# Create DataLoaders using the parameters from data prep
-subset_loaders = make_dataset_loaders(
-    subset_dataset, seed, batch_size, test_size, aug_config
-)
-
-print("\n--- Handoff Test Successful ---")
-print(f"Prototype Train loader batches: {len(subset_loaders['train'])}")
-print(f"Prototype Validation loader batches: {len(subset_loaders['val'])}")
-print(f"Prototype Test loader batches: {len(subset_loaders['test'])}")
-
-
-full_loaders = make_dataset_loaders(
-    full_dataset, seed, batch_size, test_size, aug_config
-)
-
-print("\n--- Handoff Test Successful ---")
-print(f"Train loader batches: {len(full_loaders['train'])}")
-print(f"Validation loader batches: {len(full_loaders['val'])}")
-print(f"Test loader batches: {len(full_loaders['test'])}")
-# -------------- DATA PREP ENDS --------------
+# -------------- Load Data --------------
+project_name = "Small Group Project"
+subset_loaders, full_loaders, data_prep_metadata = extract_latest_data_task(project_name=project_name)
 
 
 # -------- ClearML Training Task Setup --------
@@ -109,15 +19,16 @@ training_task = Task.init(
     reuse_last_task_id=False,
 )
 
+# Detail the data prep task used
 training_logger = training_task.get_logger()
-training_task.connect({"data_prep_task_used": DYNAMIC_TASK_ID})
+training_task.connect(data_prep_metadata, name="data_prep_metadata_READONLY")
 
 # Training parameters - Modify these to experiment
 training_config = {
     "num_classes": 39,
-    "n_epochs": 1,
+    "n_epochs": 10,
     "learning_rate": 1e-3,
-    "batch_size": batch_size,
+    "optimizer": "adam",
     "save_path": "best_model.pt",
 }
 training_task.connect(training_config)
@@ -130,37 +41,37 @@ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 # ------- Train the model (on subset for now) -------
  
-<<<<<<< HEAD
 print("\n--- Starting Model Training on Subset ---")
 training_metrics = train_model(
-=======
-#When calling this function, the model should be trained on the given dataset
-
-print("\n--- Starting Model Training on Subset ---")
-train_model(
->>>>>>> 20050ad82ebca27a376e15837a7abf79fca23e98
     model=model,
     train_loader=subset_loaders['train'],
     val_loader=subset_loaders['val'],
     device=device,
     n_epochs=training_config["n_epochs"],
     lr=training_config["learning_rate"],
+    num_classes=training_config["num_classes"],
+    optimizer_type=training_config["optimizer"],
     save_path=training_config["save_path"],
 )
-<<<<<<< HEAD
 
 
 # ----------- Log metrics to ClearML -----------
 # Per-batch training losses and accuracies
-for i, loss in enumerate(training_metrics["losses"]):
-    training_logger.report_scalar("train", "loss_per_batch", value=loss, iteration=i)
+for i, loss in enumerate(training_metrics["batch_losses"]):
+    training_logger.report_scalar("training batch loss", "loss", value=loss, iteration=i)
+
+for i, acc in enumerate(training_metrics["batch_accuracies"]):
+    training_logger.report_scalar("training batch accuracy", "accuracy", value=acc, iteration=i)
 
-for i, acc in enumerate(training_metrics["accuracies"]):
-    training_logger.report_scalar("train", "accuracy_per_batch", value=acc, iteration=i)
+# Per-epoch training losses and accuracies
+epoch_metrics = zip(training_metrics["epoch_losses"], training_metrics["epoch_accuracies"])
+for epoch, (loss, acc) in enumerate(epoch_metrics):
+    training_logger.report_scalar("training epoch loss", "loss", loss, iteration=epoch)
+    training_logger.report_scalar("training epoch accuracy", "accuracy", acc, iteration=epoch)
 
-# Per-epoch validation accuracy
+# Per-epoch validation accuracies
 for epoch, acc in enumerate(training_metrics["val_accuracies"]):
-    training_logger.report_scalar("validation", "accuracy_per_epoch", value=acc, iteration=epoch)
+    training_logger.report_scalar("validation epoch accuracy", "accuracy", value=acc, iteration=epoch)
 
 training_logger.report_single_value("best_val_accuracy", training_metrics["best_accuracy"])
 
@@ -168,6 +79,4 @@ training_logger.report_single_value("best_val_accuracy", training_metrics["best_
 training_task.upload_artifact("best_model", training_config["save_path"])
 
 print("\nTraining complete.")
-training_task.close()
-=======
->>>>>>> 20050ad82ebca27a376e15837a7abf79fca23e98
+training_task.close()