EquiFusion / src /utils /tensorboard.py
ceccurr's picture
EquiFusion demo
e6d357a verified
Raw History Blame Contribute Delete
6.09 kB
import torch
from ignite.handlers.tensorboard_logger import TensorboardLogger, WeightsScalarHandler, WeightsHistHandler, GradsHistHandler, GradsScalarHandler
from ignite.engine import Engine, Events
from torch.optim import Optimizer
def setup_tb_logging_valid(tb_logger: TensorboardLogger, trainer: Engine, evaluator: Engine, evaluator_train: Engine, evaluator_generalization: Engine,
loss_names=["Loss"], metric_names=[ "ADE", "FDE", "MPJPE_AVG", "MPJPE"], if_consider_hip=False):
if if_consider_hip:
metric_names += ['ADE_POSE', 'ADE_HIP'] if 'ADE' in metric_names else []
metric_names += ['FDE_POSE', 'FDE_HIP'] if 'FDE' in metric_names else []
metric_names += ['MPJPE_POSE', "MPJPE_HIP"] if 'MPJPE' in metric_names else []
loss_names += ['Loss Pose'] if 'Loss' in loss_names else [] # , 'Loss Hip'
tb_custom_scalar_layout = {}
# We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the
# `trainer` instead of `evaluator`.
tb_logger.attach_output_handler(
evaluator_train,
event_name=Events.COMPLETED,
**{'tag': "losses/train",
'metric_names': loss_names,
'global_step_transform': lambda engine, event_name: trainer.state.epoch
}
)
tb_logger.attach_output_handler(
evaluator,
event_name=Events.COMPLETED,
**{'tag': "losses/validation",
'metric_names': loss_names,
'global_step_transform': lambda engine, event_name: trainer.state.epoch
}
)
tb_logger.attach_output_handler(
evaluator,
event_name=Events.COMPLETED,
**{'tag': "validation",
'metric_names': loss_names + metric_names,
'global_step_transform': lambda engine, event_name: trainer.state.epoch
}
)
tb_logger.attach_output_handler(
evaluator_train,
event_name=Events.COMPLETED,
**{'tag': "validation_train",
'metric_names': loss_names + metric_names,
'global_step_transform': lambda engine, event_name: trainer.state.epoch
}
)
tb_logger.attach_output_handler(
evaluator_generalization,
event_name=Events.COMPLETED,
**{'tag': "validation_generalization",
'metric_names': loss_names + metric_names,
'global_step_transform': lambda engine, event_name: trainer.state.epoch
}
)
tb_custom_scalar_layout = {
**tb_custom_scalar_layout,
**{
'Validation Metrics': {
'MPJPE_T': ['Multiline', [rf"validation_train/MPJPE_T/.*"]],
}
}
}
tb_logger.writer.add_custom_scalars(tb_custom_scalar_layout)
def setup_tb_logging_train(tb_logger: TensorboardLogger, trainer: Engine, optimizer: Optimizer,
model: torch.nn.Module):
"""
Logs standard information to tensorboard during training.
"""
# Attach the logger to the trainer to log training loss at each iteration
def _setup_tb_train_curves(
**kwargs,
):
# trainer returns loss, (loss_dict, loss_weights, raw_losses, grad_norms), pred, y, None
def funct_losses(out):
return out[1][0]
def funct_weights(out):
return out[1][1]
def funct_raw_losses(out):
return out[1][2]
def funct_grad_norms(out):
return out[1][3]
return funct_losses, funct_weights, funct_raw_losses, funct_grad_norms
funct_losses, funct_weights, funct_raw_losses, funct_grad_norms = _setup_tb_train_curves()
tb_logger.attach_output_handler(
trainer,
event_name=Events.ITERATION_COMPLETED,
**{
'tag': 'training',
'output_transform': funct_losses,
}
)
tb_logger.attach_output_handler(
trainer,
event_name=Events.ITERATION_COMPLETED,
**{
'tag': 'losses_raw',
'output_transform': funct_raw_losses,
}
)
tb_logger.attach_output_handler(
trainer,
event_name=Events.ITERATION_COMPLETED,
**{
'tag': 'loss_weights',
'output_transform': funct_weights,
}
)
tb_logger.attach_output_handler(
trainer,
event_name=Events.ITERATION_COMPLETED,
**{
'tag': 'grad_norms',
'output_transform': funct_grad_norms,
}
)
# Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration
tb_logger.attach_opt_params_handler(
trainer,
event_name=Events.ITERATION_COMPLETED,
**{
'optimizer': optimizer,
'param_name': 'lr',
'tag': 'training'
}
)
# Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration
tb_logger.attach_opt_params_handler(
trainer,
event_name=Events.EPOCH_STARTED,
**{
'optimizer': optimizer,
'param_name': 'lr',
'tag': 'losses'
}
)
# Attach the logger to the trainer to log model's weights norm after each iteration
tb_logger.attach(
trainer,
event_name=Events.ITERATION_COMPLETED(every=100),
log_handler=WeightsScalarHandler(model)
)
# Attach the logger to the trainer to log model's weights as a histogram after each epoch
tb_logger.attach(
trainer,
event_name=Events.EPOCH_COMPLETED,
log_handler=WeightsHistHandler(model)
)
# Attach the logger to the trainer to log model's gradients norm after each iteration
tb_logger.attach(
trainer,
event_name=Events.ITERATION_COMPLETED(every=100),
log_handler=GradsScalarHandler(model)
)
# Attach the logger to the trainer to log model's gradients as a histogram after each epoch
tb_logger.attach(
trainer,
event_name=Events.EPOCH_COMPLETED,
log_handler=GradsHistHandler(model)
)