Spaces:
Sleeping
Sleeping
Download loomnet/utils/misc.py from FedericoGiulio/LoomNet: direct link, hf CLI and curl.
- Browser
- Download file 6.3 kB
-
https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/utils/misc.py
- Command line
-
hf download hf://spaces/FedericoGiulio/LoomNet/loomnet/utils/misc.py
-
curl -L -o misc.py https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/utils/misc.py
6.3 kB
| import os | |
| import re | |
| import warnings | |
| from importlib.util import find_spec | |
| from typing import Callable, Optional, Tuple | |
| import torch | |
| from lightning import Trainer | |
| from omegaconf import DictConfig | |
| from loomnet.utils import RankedLogger | |
| log = RankedLogger(__name__, rank_zero_only=True) | |
| def count_params(model, verbose=False): | |
| total_params = sum(p.numel() for p in model.parameters()) | |
| if verbose: | |
| print(f"{model.__class__.__name__} has {total_params*1.e-6:.2f} M params.") | |
| return total_params | |
| def get_rank(): | |
| # SLURM_PROCID can be set even if SLURM is not managing the multiprocessing, | |
| # therefore LOCAL_RANK needs to be checked first | |
| rank_keys = ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK") | |
| for key in rank_keys: | |
| rank = os.environ.get(key) | |
| if rank is not None: | |
| return int(rank) | |
| return 0 | |
| def get_device(): | |
| return torch.device(f"cuda:{get_rank()}") | |
| def load_module_weights( | |
| path, module_name=None, ignore_modules=None, map_location=None | |
| ) -> Tuple[dict, int, int]: | |
| """Load module weights from a checkpoint. | |
| This method is useful when you want to load weights from a checkpoint. You can specify | |
| a module name, and only the weights of that module will be loaded. You can also specify | |
| a list of modules to ignore. | |
| :param path: Path to the checkpoint. | |
| :param module_name: Name of the module to load. | |
| :param ignore_modules: List of modules to ignore. | |
| :param map_location: Map location for the checkpoint. Defaults to the current device. | |
| :return: A tuple containing the state dict, the epoch and the global step. | |
| """ | |
| if module_name is not None and ignore_modules is not None: | |
| raise ValueError("module_name and ignore_modules cannot be both set") | |
| if map_location is None: | |
| map_location = get_device() | |
| ckpt = torch.load(path, map_location=map_location, weights_only=False) | |
| state_dict = ckpt["state_dict"] | |
| state_dict_to_load = state_dict | |
| if ignore_modules is not None: | |
| state_dict_to_load = {} | |
| for k, v in state_dict.items(): | |
| ignore = any( | |
| [k.startswith(ignore_module + ".") for ignore_module in ignore_modules] | |
| ) | |
| if ignore: | |
| continue | |
| state_dict_to_load[k] = v | |
| if module_name is not None: | |
| state_dict_to_load = {} | |
| for k, v in state_dict.items(): | |
| m = re.match(rf"^{module_name}\.(.*)$", k) | |
| if m is None: | |
| continue | |
| state_dict_to_load[m.group(1)] = v | |
| return state_dict_to_load, ckpt["epoch"], ckpt["global_step"] | |
| def resolve_dir( | |
| trainer: Trainer, | |
| dirpath: Optional[str] = None, | |
| sub_dir: Optional[str] = None, | |
| ) -> str: | |
| """Determines save directory at runtime. Reference attributes from the trainer's logger to | |
| determine where to save. The path for saving weights is set in this priority: | |
| 1. The ``dirpath`` if passed in | |
| 2. The ``Logger``'s ``log_dir`` if the trainer has loggers | |
| 3. The ``Trainer``'s ``default_root_dir`` if the trainer has no loggers | |
| The path gets extended with subdirectory ``name`` if passed in. | |
| """ | |
| if dirpath is not None: | |
| # short circuit if dirpath was passed to ModelCheckpoint | |
| return dirpath | |
| if len(trainer.loggers) > 0: | |
| if trainer.loggers[0].save_dir is not None: | |
| save_dir = trainer.loggers[0].save_dir | |
| else: | |
| save_dir = trainer.default_root_dir | |
| _name = trainer.loggers[0]._name | |
| version = trainer.loggers[0].version | |
| version = version if isinstance(version, str) else f"version_{version}" | |
| dirpath = os.path.join(save_dir, _name or version) | |
| else: | |
| # if no loggers, use default_root_dir | |
| dirpath = trainer.default_root_dir | |
| if sub_dir is not None: | |
| dirpath = os.path.join(dirpath, sub_dir) | |
| return dirpath | |
| def apply_extras(cfg: DictConfig) -> None: | |
| """Applies optional utilities before the task is started. | |
| Utilities: | |
| - Ignoring python warnings | |
| - ... | |
| :param cfg: A DictConfig object containing the config tree. | |
| """ | |
| # return if no `extras` config | |
| if not cfg.get("extras"): | |
| log.warning("Extras config not found! <cfg.extras=null>") | |
| return | |
| # disable python warnings | |
| if cfg.extras.get("ignore_warnings"): | |
| log.info("Disabling python warnings! <cfg.extras.ignore_warnings=True>") | |
| warnings.filterwarnings("ignore") | |
| def task_wrapper(task_func: Callable) -> Callable: | |
| """Optional decorator that controls the failure behavior when executing the task function. | |
| This wrapper can be used to: | |
| - make sure loggers are closed even if the task function raises an exception (prevents multirun failure) | |
| - save the exception to a `.log` file | |
| - mark the run as failed with a dedicated file in the `logs/` folder (so we can find and rerun it later) | |
| - etc. (adjust depending on your needs) | |
| Example: | |
| ``` | |
| @utils.task_wrapper | |
| def train(**kwargs): | |
| ... | |
| return | |
| ``` | |
| :param task_func: The task function to be wrapped. | |
| :return: The wrapped task function. | |
| """ | |
| def wrap(cfg, config_path=None): | |
| # execute the task | |
| try: | |
| task_func(cfg, config_path=config_path) | |
| # things to do if exception occurs | |
| except Exception as ex: | |
| # save exception to `.log` file | |
| log.exception("") | |
| # some hyperparameter combinations might be invalid or cause out-of-memory errors | |
| # so when using hparam search plugins like Optuna, you might want to disable | |
| # raising the below exception to avoid multirun failure | |
| raise ex | |
| # things to always do after either success or exception | |
| finally: | |
| # display output dir path in terminal | |
| log.info(f"Output dir: {cfg.output_dir}") | |
| # always close wandb run (even if exception occurs so multirun won't fail) | |
| if find_spec("wandb"): # check if wandb is installed | |
| import wandb | |
| if wandb.run: | |
| log.info("Closing wandb!") | |
| wandb.finish() | |
| return wrap | |