feat: clearml training metrics

trainingModel/Training.py

@@ -139,5 +139,12 @@ def train_model(
     print(f"\nTraining finished. Best val accuracy: {best_accuracy:.4f}")
     print(f"Best model weights saved to: {save_path}")
 
-    return training_losses, training_accuracies, val_accuracies, best_accuracy
+    training_metrics = {
+        "losses": training_losses,
+        "accuracies": training_accuracies,
+        "val_accuracies": val_accuracies,
+        "best_accuracy": best_accuracy,
+    }
+
+    return training_metrics
 

trainingModel/run_training.py

@@ -10,7 +10,8 @@ from models.modelOne import modelOne
 from trainingModel.Training import train_model
 
 
-# Latest Data Prep Task 
+# -------------- Load Data --------------
+
 all_tasks = Task.get_tasks(project_name="Small Group Project")
 if not all_tasks:
     raise RuntimeError("No tasks found in project 'Small Group Project'")
@@ -19,6 +20,7 @@ dp_tasks = [t for t in all_tasks if t.name == "Data Preparation"]
 if not dp_tasks:
     raise RuntimeError("No 'Data Preparation' tasks found in this project!")
 
+# Latest Data Prep Task 
 latest_task = max(dp_tasks, key=lambda t: t.id)
 DYNAMIC_TASK_ID = latest_task.id
 DATA_PREP = Task.get_task(task_id=DYNAMIC_TASK_ID)
@@ -32,8 +34,7 @@ dataset_id = raw_meta.split("=")[1].strip().replace('"', "")
 subset_clearml = Dataset.get(dataset_id=dataset_id)
 local_folder = subset_clearml.get_local_copy()
 
-subset_indices_path = os.path.join(local_folder, "subset_indices.npy")
-subset_indices = np.load(subset_indices_path)
+subset_indices = np.load(os.path.join(local_folder, "subset_indices.npy"))
 
 # Load Dataset Parameters 
 data_params = DATA_PREP.get_parameters()
@@ -84,24 +85,65 @@ print("\n--- Handoff Test Successful ---")
 print(f"Train loader batches: {len(full_loaders['train'])}")
 print(f"Validation loader batches: {len(full_loaders['val'])}")
 print(f"Test loader batches: {len(full_loaders['test'])}")
+# -------------- DATA PREP ENDS --------------
+
+
+# -------- ClearML Training Task Setup --------
+training_task = Task.init(
+    project_name="Small Group Project",
+    task_name="Model Training",
+    reuse_last_task_id=False,
+)
+
+training_logger = training_task.get_logger()
+training_task.connect({"data_prep_task_used": DYNAMIC_TASK_ID})
+
+# Training parameters - Modify these to experiment
+training_config = {
+    "num_classes": 39,
+    "n_epochs": 1,
+    "learning_rate": 1e-3,
+    "batch_size": batch_size,
+    "save_path": "best_model.pt",
+}
+training_task.connect(training_config)
 
 
 # -------- Build the ML model --------
-model = modelOne(noOfClasses=39)
+model = modelOne(noOfClasses=training_config["num_classes"])
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
 # ------- Train the model (on subset for now) -------
  
-#When calling this function, the model should be trained on the given dataset
-
 print("\n--- Starting Model Training on Subset ---")
-train_model(
+training_metrics = train_model(
     model=model,
     train_loader=subset_loaders['train'],
     val_loader=subset_loaders['val'],
     device=device,
-    n_epochs=10,
-    lr=1e-3,
-    save_path="best_model.pt",
-)
+    n_epochs=training_config["n_epochs"],
+    lr=training_config["learning_rate"],
+    save_path=training_config["save_path"],
+)
+
+
+# ----------- Log metrics to ClearML -----------
+# Per-batch training losses and accuracies
+for i, loss in enumerate(training_metrics["losses"]):
+    training_logger.report_scalar("train", "loss_per_batch", value=loss, iteration=i)
+
+for i, acc in enumerate(training_metrics["accuracies"]):
+    training_logger.report_scalar("train", "accuracy_per_batch", value=acc, iteration=i)
+
+# Per-epoch validation accuracy
+for epoch, acc in enumerate(training_metrics["val_accuracies"]):
+    training_logger.report_scalar("validation", "accuracy_per_epoch", value=acc, iteration=epoch)
+
+training_logger.report_single_value("best_val_accuracy", training_metrics["best_accuracy"])
+
+# Upload best model as artifact
+training_task.upload_artifact("best_model", training_config["save_path"])
+
+print("\nTraining complete.")
+training_task.close()