LoomNet / loomnet /utils /misc.py
Giulio Federico
Upload of the code without weights
2039c18
Raw History Blame Contribute Delete
6.3 kB
import os
import re
import warnings
from importlib.util import find_spec
from typing import Callable, Optional, Tuple
import torch
from lightning import Trainer
from omegaconf import DictConfig
from loomnet.utils import RankedLogger
log = RankedLogger(__name__, rank_zero_only=True)
def count_params(model, verbose=False):
total_params = sum(p.numel() for p in model.parameters())
if verbose:
print(f"{model.__class__.__name__} has {total_params*1.e-6:.2f} M params.")
return total_params
def get_rank():
# SLURM_PROCID can be set even if SLURM is not managing the multiprocessing,
# therefore LOCAL_RANK needs to be checked first
rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")
for key in rank_keys:
rank = os.environ.get(key)
if rank is not None:
return int(rank)
return 0
def get_device():
return torch.device(f"cuda:{get_rank()}")
def load_module_weights(
path, module_name=None, ignore_modules=None, map_location=None
) -> Tuple[dict, int, int]:
"""Load module weights from a checkpoint.
This method is useful when you want to load weights from a checkpoint. You can specify
a module name, and only the weights of that module will be loaded. You can also specify
a list of modules to ignore.
:param path: Path to the checkpoint.
:param module_name: Name of the module to load.
:param ignore_modules: List of modules to ignore.
:param map_location: Map location for the checkpoint. Defaults to the current device.
:return: A tuple containing the state dict, the epoch and the global step.
"""
if module_name is not None and ignore_modules is not None:
raise ValueError("module_name and ignore_modules cannot be both set")
if map_location is None:
map_location = get_device()
ckpt = torch.load(path, map_location=map_location, weights_only=False)
state_dict = ckpt["state_dict"]
state_dict_to_load = state_dict
if ignore_modules is not None:
state_dict_to_load = {}
for k, v in state_dict.items():
ignore = any(
[k.startswith(ignore_module + ".") for ignore_module in ignore_modules]
)
if ignore:
continue
state_dict_to_load[k] = v
if module_name is not None:
state_dict_to_load = {}
for k, v in state_dict.items():
m = re.match(rf"^{module_name}\.(.*)$", k)
if m is None:
continue
state_dict_to_load[m.group(1)] = v
return state_dict_to_load, ckpt["epoch"], ckpt["global_step"]
def resolve_dir(
trainer: Trainer,
dirpath: Optional[str] = None,
sub_dir: Optional[str] = None,
) -> str:
"""Determines save directory at runtime. Reference attributes from the trainer's logger to
determine where to save. The path for saving weights is set in this priority:
1. The ``dirpath`` if passed in
2. The ``Logger``'s ``log_dir`` if the trainer has loggers
3. The ``Trainer``'s ``default_root_dir`` if the trainer has no loggers
The path gets extended with subdirectory ``name`` if passed in.
"""
if dirpath is not None:
# short circuit if dirpath was passed to ModelCheckpoint
return dirpath
if len(trainer.loggers) > 0:
if trainer.loggers[0].save_dir is not None:
save_dir = trainer.loggers[0].save_dir
else:
save_dir = trainer.default_root_dir
_name = trainer.loggers[0]._name
version = trainer.loggers[0].version
version = version if isinstance(version, str) else f"version_{version}"
dirpath = os.path.join(save_dir, _name or version)
else:
# if no loggers, use default_root_dir
dirpath = trainer.default_root_dir
if sub_dir is not None:
dirpath = os.path.join(dirpath, sub_dir)
return dirpath
def apply_extras(cfg: DictConfig) -> None:
"""Applies optional utilities before the task is started.
Utilities:
- Ignoring python warnings
- ...
:param cfg: A DictConfig object containing the config tree.
"""
# return if no `extras` config
if not cfg.get("extras"):
log.warning("Extras config not found! <cfg.extras=null>")
return
# disable python warnings
if cfg.extras.get("ignore_warnings"):
log.info("Disabling python warnings! <cfg.extras.ignore_warnings=True>")
warnings.filterwarnings("ignore")
def task_wrapper(task_func: Callable) -> Callable:
"""Optional decorator that controls the failure behavior when executing the task function.
This wrapper can be used to:
- make sure loggers are closed even if the task function raises an exception (prevents multirun failure)
- save the exception to a `.log` file
- mark the run as failed with a dedicated file in the `logs/` folder (so we can find and rerun it later)
- etc. (adjust depending on your needs)
Example:
```
@utils.task_wrapper
def train(**kwargs):
...
return
```
:param task_func: The task function to be wrapped.
:return: The wrapped task function.
"""
def wrap(cfg, config_path=None):
# execute the task
try:
task_func(cfg, config_path=config_path)
# things to do if exception occurs
except Exception as ex:
# save exception to `.log` file
log.exception("")
# some hyperparameter combinations might be invalid or cause out-of-memory errors
# so when using hparam search plugins like Optuna, you might want to disable
# raising the below exception to avoid multirun failure
raise ex
# things to always do after either success or exception
finally:
# display output dir path in terminal
log.info(f"Output dir: {cfg.output_dir}")
# always close wandb run (even if exception occurs so multirun won't fail)
if find_spec("wandb"): # check if wandb is installed
import wandb
if wandb.run:
log.info("Closing wandb!")
wandb.finish()
return wrap