File size: 1,357 Bytes
4754feb | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | from dataclasses import asdict
import torch
from omegaconf import OmegaConf
def get_seq_idx(cu_seqlens, device=None):
seq_idx = torch.zeros(cu_seqlens[-1], dtype=torch.long, device=device)
seq_idx[cu_seqlens[:-1]] = 1
seq_idx = (torch.cumsum(seq_idx, dim=0) - 1).unsqueeze(0).int()
return seq_idx
def get_stage_cfg(cfg, stage_idx):
def dictify(cfg):
if type(cfg) is dict:
return cfg
elif OmegaConf.is_dict(cfg):
return OmegaConf.to_container(cfg, resolve=True)
return asdict(cfg)
return {
k: v[stage_idx] if isinstance(v, list) else v for k, v in dictify(cfg).items()
}
def apply_optimization_params(
param: torch.Tensor,
**kwargs,
) -> None:
"""
Annotates a parameter with optimization parameters.
Specifically, updates the parameter's `_optim` attribute with the given kwargs.
"""
if hasattr(param, "_optim"):
param._optim.update(kwargs)
else:
param._optim = kwargs
class FlopsCounter:
def __init__(self, device):
self.flops_used = torch.tensor(0.0, device=device)
self.reset()
def add_flops(self, flops: torch.FloatTensor):
self.flops_used += flops
def get_flops(self):
return self.flops_used
def reset(self):
self.flops_used = self.flops_used * 0.0
|