import inspect import json import os import sys from os.path import basename, dirname, expanduser, isdir, isfile, join, realpath from shutil import copy import torch import yaml class Logger(object): def __getattr__(self, k): return print log = Logger() def training_config_from_cli_args(): experiment_name = sys.argv[1] experiment_id = int(sys.argv[2]) yaml_config = yaml.load(open(f"experiments/{experiment_name}"), Loader=yaml.SafeLoader) config = yaml_config["configuration"] config = {**config, **yaml_config["individual_configurations"][experiment_id]} config = AttributeDict(config) return config def score_config_from_cli_args(): experiment_name = sys.argv[1] experiment_id = int(sys.argv[2]) yaml_config = yaml.load(open(experiment_name), Loader=yaml.SafeLoader) config = yaml_config["test_configuration_common"] config = {**config, **yaml_config["test_configuration"]} if "test_configuration" in yaml_config["individual_configurations"][experiment_id]: config = {**config, **yaml_config["individual_configurations"][experiment_id]["test_configuration"]} train_checkpoint_id = yaml_config["individual_configurations"][experiment_id]["name"] config["pred_json_root"] = sys.argv[3] config = AttributeDict(config) return config, train_checkpoint_id def get_from_repository( local_name, repo_files, integrity_check=None, repo_dir="~/dataset_repository", local_dir="~/datasets" ): """copies files from repository to local folder. repo_files: list of filenames or list of tuples [filename, target path] e.g. get_from_repository('MyDataset', [['data/dataset1.tar', 'other/path/ds03.tar']) will create a folder 'MyDataset' in local_dir, and extract the content of '/data/dataset1.tar' to /MyDataset/other/path. """ local_dir = realpath(join(expanduser(local_dir), local_name)) dataset_exists = True # check if folder is available if not isdir(local_dir): dataset_exists = False if integrity_check is not None: try: integrity_ok = integrity_check(local_dir) except BaseException: integrity_ok = False if integrity_ok: log.hint("Passed custom integrity check") else: log.hint("Custom integrity check failed") dataset_exists = dataset_exists and integrity_ok if not dataset_exists: repo_dir = realpath(expanduser(repo_dir)) for i, filename in enumerate(repo_files): if type(filename) == str: origin, target = filename, filename archive_target = join(local_dir, basename(origin)) extract_target = join(local_dir) else: origin, target = filename archive_target = join(local_dir, dirname(target), basename(origin)) extract_target = join(local_dir, dirname(target)) archive_origin = join(repo_dir, origin) log.hint(f"copy: {archive_origin} to {archive_target}") # make sure the path exists os.makedirs(dirname(archive_target), exist_ok=True) if os.path.isfile(archive_target): # only copy if size differs if os.path.getsize(archive_target) != os.path.getsize(archive_origin): log.hint( f"file exists but filesize differs: target {os.path.getsize(archive_target)} vs. origin {os.path.getsize(archive_origin)}" ) copy(archive_origin, archive_target) else: copy(archive_origin, archive_target) extract_archive(archive_target, extract_target, noarchive_ok=True) # concurrent processes might have deleted the file if os.path.isfile(archive_target): os.remove(archive_target) def extract_archive(filename, target_folder=None, noarchive_ok=False): from subprocess import PIPE, run if filename.endswith(".tgz") or filename.endswith(".tar"): command = f"tar -xf {filename}" command += f" -C {target_folder}" if target_folder is not None else "" elif filename.endswith(".tar.gz"): command = f"tar -xzf {filename}" command += f" -C {target_folder}" if target_folder is not None else "" elif filename.endswith("zip"): command = f"unzip {filename}" command += f" -d {target_folder}" if target_folder is not None else "" else: if noarchive_ok: return else: raise ValueError(f"unsuppored file ending of {filename}") log.hint(command) result = run(command.split(), stdout=PIPE, stderr=PIPE) if result.returncode != 0: print(result.stdout, result.stderr) class AttributeDict(dict): """ An extended dictionary that allows access to elements as atttributes and counts these accesses. This way, we know if some attributes were never used. """ def __init__(self, *args, **kwargs): from collections import Counter super().__init__(*args, **kwargs) self.__dict__["counter"] = Counter() def __getitem__(self, k): self.__dict__["counter"][k] += 1 return super().__getitem__(k) def __getattr__(self, k): self.__dict__["counter"][k] += 1 return super().get(k) def __setattr__(self, k, v): return super().__setitem__(k, v) def __delattr__(self, k, v): return super().__delitem__(k, v) def unused_keys(self, exceptions=()): return [k for k in super().keys() if self.__dict__["counter"][k] == 0 and k not in exceptions] def assume_no_unused_keys(self, exceptions=()): if len(self.unused_keys(exceptions=exceptions)) > 0: log.warning("Unused keys:", self.unused_keys(exceptions=exceptions)) def get_attribute(name): import importlib if name is None: raise ValueError("The provided attribute is None") name_split = name.split(".") mod = importlib.import_module(".".join(name_split[:-1])) return getattr(mod, name_split[-1]) def filter_args(input_args, default_args): updated_args = {k: input_args[k] if k in input_args else v for k, v in default_args.items()} used_args = {k: v for k, v in input_args.items() if k in default_args} unused_args = {k: v for k, v in input_args.items() if k not in default_args} return AttributeDict(updated_args), AttributeDict(used_args), AttributeDict(unused_args) def load_model(checkpoint_id, weights_file=None, strict=True, model_args="from_config", with_config=False): config = json.load(open(join("logs", checkpoint_id, "config.json"))) if model_args != "from_config" and type(model_args) != dict: raise ValueError('model_args must either be "from_config" or a dictionary of values') model_cls = get_attribute(config["model"]) # load model if model_args == "from_config": _, model_args, _ = filter_args(config, inspect.signature(model_cls).parameters) model = model_cls(**model_args) if weights_file is None: weights_file = realpath(join("logs", checkpoint_id, "weights.pth")) else: weights_file = realpath(join("logs", checkpoint_id, weights_file)) if isfile(weights_file): weights = torch.load(weights_file) for _, w in weights.items(): assert not torch.any(torch.isnan(w)), "weights contain NaNs" model.load_state_dict(weights, strict=strict) else: raise FileNotFoundError(f"model checkpoint {weights_file} was not found") if with_config: return model, config return model class TrainingLogger(object): def __init__(self, model, log_dir, config=None, *args): super().__init__() self.model = model self.base_path = join(f"logs/{log_dir}") if log_dir is not None else None os.makedirs("logs/", exist_ok=True) os.makedirs(self.base_path, exist_ok=True) if config is not None: json.dump(config, open(join(self.base_path, "config.json"), "w")) def iter(self, i, **kwargs): if i % 100 == 0 and "loss" in kwargs: loss = kwargs["loss"] print(f"iteration {i}: loss {loss:.4f}") def save_weights(self, only_trainable=False, weight_file="weights.pth"): if self.model is None: raise AttributeError( "You need to provide a model reference when initializing TrainingTracker to save weights." ) weights_path = join(self.base_path, weight_file) weight_dict = self.model.state_dict() if only_trainable: weight_dict = {n: weight_dict[n] for n, p in self.model.named_parameters() if p.requires_grad} torch.save(weight_dict, weights_path) log.info(f"Saved weights to {weights_path}") def __enter__(self): return self def __exit__(self, type, value, traceback): """automatically stop processes if used in a context manager""" pass