cosmos-framework/cosmos_framework/callbacks/wandb_log_eval.py

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: OpenMDW-1.1

from __future__ import annotations

from dataclasses import dataclass
from typing import Tuple

import torch
import torch.distributed as dist
import torch.utils.data
import wandb

from cosmos_framework.model._base import ImaginaireModel
from cosmos_framework.utils import distributed, log
from cosmos_framework.utils.callback import Callback
from cosmos_framework.utils.easy_io import easy_io


@dataclass
class _LossRecord:
    loss: float = 0
    iter_count: int = 0

    def reset(self) -> None:
        self.loss = 0
        self.iter_count = 0

    def get_stat(self) -> Tuple[float, float]:
        if self.iter_count > 0:
            avg_loss_tensor = self.loss / self.iter_count
            # Create a mask (1 if valid, 0 if NaN or Inf)
            valid_mask = torch.tensor([torch.isfinite(avg_loss_tensor).float()], device="cuda")

            # Replace NaN/Inf with 0 to avoid affecting sum
            avg_loss_tensor = torch.where(
                torch.isfinite(avg_loss_tensor), avg_loss_tensor, torch.tensor([0.0], device="cuda")
            )

            # Reduce across all ranks
            dist.all_reduce(avg_loss_tensor, op=dist.ReduceOp.SUM)  # Sum of valid losses
            dist.all_reduce(valid_mask, op=dist.ReduceOp.SUM)  # Count of valid losses

            # Compute final average, avoiding division by zero
            if valid_mask.item() > 0:
                final_avg_loss = (avg_loss_tensor / valid_mask).item()
            else:
                final_avg_loss = 0.0  # Default to zero if all values were invalid

            avg_loss = final_avg_loss
        else:
            avg_loss = 0
        self.reset()
        return avg_loss


class WandbCallback(Callback):
    def __init__(
        self,
        save_s3: bool = False,
    ) -> None:
        super().__init__()
        self.final_loss_log = _LossRecord()
        self.final_loss_log_per_dataset = {}

        self.save_s3 = save_s3
        self.wandb_extra_tag = ""
        self.name = "wandb_loss_val_log"
        self.unstable_count = torch.zeros(1, device="cuda")
        self.url_key_list = []

    def on_validation_step_end(
        self,
        model: ImaginaireModel,
        data_batch: dict[str, torch.Tensor],
        output_batch: dict[str, torch.Tensor],
        loss: torch.Tensor,
        iteration: int = 0,
    ) -> None:
        if torch.isnan(loss) or torch.isinf(loss):
            log.critical(
                f"Unstable val loss {loss} at iteration {iteration}",
                rank0_only=False,
            )
            self.unstable_count += 1

        dataset_name = data_batch.get("dataset_name", "default")

        # Handle case where dataset_name gets batched into a list
        if isinstance(dataset_name, list):

            assert len(dataset_name) == 1, "dataset_name should be a list of 1"
            dataset_name = dataset_name[0]

        if dataset_name not in self.final_loss_log_per_dataset:
            self.final_loss_log_per_dataset[dataset_name] = _LossRecord()

        self.final_loss_log_per_dataset[dataset_name].loss += loss.detach().float()
        self.final_loss_log_per_dataset[dataset_name].iter_count += 1
        self.final_loss_log.loss += loss.detach().float()
        self.final_loss_log.iter_count += 1

        self.url_key_list.append(f"{data_batch.get('__url__', [''])[0]}, {data_batch.get('__key__', [''])[0]}")

    def on_validation_end(self, model: ImaginaireModel, iteration: int = 0) -> None:
        avg_final_loss = self.final_loss_log.get_stat()

        log.info(f"avg_final_loss: {avg_final_loss}")
        dist.all_reduce(self.unstable_count, op=dist.ReduceOp.SUM)
        # gather url and key list from all ranks
        url_key_list = [None] * dist.get_world_size()
        dist.all_gather_object(url_key_list, self.url_key_list)
        url_key_list = [item for sublist in url_key_list for item in sublist]

        unique_url_key_list = list(set(url_key_list))
        if distributed.is_rank0():
            info = {}
            log.info(
                f"[val] number of unique url and key: {len(unique_url_key_list)} / {len(url_key_list)}; avg_final_loss: {avg_final_loss}"
            )
            info.update(
                {
                    f"val{self.wandb_extra_tag}/loss": avg_final_loss,
                    f"val{self.wandb_extra_tag}/unstable_count": self.unstable_count.item(),
                    "iteration": iteration,
                    f"val{self.wandb_extra_tag}/num_unique_url_key": len(unique_url_key_list),
                    f"val{self.wandb_extra_tag}/total_url_key": len(url_key_list),
                }
            )
            if self.save_s3:
                easy_io.dump(
                    info,
                    f"s3://rundir/{self.name}/Val_Iter{iteration:09d}.json",
                )

            if wandb.run is not None:
                wandb.log(info, step=iteration)

        # reset unstable count
        self.unstable_count.zero_()
        self.url_key_list = []