File size: 4,400 Bytes
d4cbafd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | import os
import torch
import numpy as np
import random
import time
import logging
import logging.handlers
THOUSAND = 1000
MILLION = 1000000
class BlackHole(object):
def __setattr__(self, name, value):
pass
def __call__(self, *args, **kwargs):
return self
def __getattr__(self, name):
return self
class CheckpointManager(object):
def __init__(self, save_dir, logger=BlackHole()):
super().__init__()
os.makedirs(save_dir, exist_ok=True)
self.save_dir = save_dir
self.ckpts = []
self.logger = logger
for f in os.listdir(self.save_dir):
if f[:4] != 'ckpt':
continue
_, score, it = f.split('_')
it = it.split('.')[0]
self.ckpts.append({
'score': float(score),
'file': f,
'iteration': int(it),
})
def get_worst_ckpt_idx(self):
idx = -1
worst = float('-inf')
for i, ckpt in enumerate(self.ckpts):
if ckpt['score'] >= worst:
idx = i
worst = ckpt['score']
return idx if idx >= 0 else None
def get_best_ckpt_idx(self):
idx = -1
best = float('inf')
for i, ckpt in enumerate(self.ckpts):
if ckpt['score'] <= best:
idx = i
best = ckpt['score']
return idx if idx >= 0 else None
def get_latest_ckpt_idx(self):
idx = -1
latest_it = -1
for i, ckpt in enumerate(self.ckpts):
if ckpt['iteration'] > latest_it:
idx = i
latest_it = ckpt['iteration']
return idx if idx >= 0 else None
def save(self, model, args, score, others=None, step=None):
if step is None:
fname = 'ckpt_%.6f_.pt' % float(score)
else:
fname = 'ckpt_%.6f_%d.pt' % (float(score), int(step))
path = os.path.join(self.save_dir, fname)
torch.save({
'args': args,
'state_dict': model.state_dict(),
'others': others
}, path)
self.ckpts.append({
'score': score,
'file': fname
})
return True
def load_best(self):
idx = self.get_best_ckpt_idx()
if idx is None:
raise IOError('No checkpoints found.')
ckpt = torch.load(os.path.join(self.save_dir, self.ckpts[idx]['file']))
return ckpt
def load_latest(self):
idx = self.get_latest_ckpt_idx()
if idx is None:
raise IOError('No checkpoints found.')
ckpt = torch.load(os.path.join(self.save_dir, self.ckpts[idx]['file']))
return ckpt
def load_selected(self, file):
ckpt = torch.load(os.path.join(self.save_dir, file))
return ckpt
def seed_all(seed):
torch.manual_seed(seed)
np.random.seed(seed)
random.seed(seed)
def get_logger(name, log_dir=None):
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('[%(asctime)s::%(name)s::%(levelname)s] %(message)s')
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.DEBUG)
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
if log_dir is not None:
file_handler = logging.FileHandler(os.path.join(log_dir, 'log.txt'))
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
return logger
def get_new_log_dir(root='./logs', postfix='', prefix=''):
log_dir = os.path.join(root, prefix + time.strftime('%Y_%m_%d__%H_%M_%S', time.localtime()) + postfix)
os.makedirs(log_dir)
return log_dir
def int_tuple(argstr):
return tuple(map(int, argstr.split(',')))
def str_tuple(argstr):
return tuple(argstr.split(','))
def int_list(argstr):
return list(map(int, argstr.split(',')))
def str_list(argstr):
return list(argstr.split(','))
def log_hyperparams(writer, args):
from torch.utils.tensorboard.summary import hparams
vars_args = {k:v if isinstance(v, str) else repr(v) for k, v in vars(args).items()}
exp, ssi, sei = hparams(vars_args, {})
writer.file_writer.add_summary(exp)
writer.file_writer.add_summary(ssi)
writer.file_writer.add_summary(sei)
|