File size: 3,082 Bytes
872b0a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | import os
import subprocess
import configparser
from shutil import rmtree
import tensorflow as tf
CONFIG_PATH = '../config.ini'
#TMP_DIR = '/tmp/e2eflow'
def upload_gdrive(upload_dir, gdrive_filename):
# search for file in gdrive and capture id if it already exists
lst_lines = subprocess.Popen(['../scripts/gdrive', 'list'],
stdout=subprocess.PIPE)
existing_id = None
for line in lst_lines.stdout:
splits = line.split()
if str(splits[1], 'utf-8') == gdrive_filename:
existing_id = str(splits[0], 'utf-8')
tmp_path = os.path.join('/tmp', gdrive_filename)
if os.path.isfile(tmp_path):
os.remove(tmp_path)
p = subprocess.Popen(['/usr/bin/zip', '-r', tmp_path, upload_dir])
p.wait()
if existing_id:
p = subprocess.Popen(['../scripts/gdrive', 'update',
existing_id, tmp_path])
else:
p = subprocess.Popen(['../scripts/gdrive', 'upload',
'--name', gdrive_filename,
tmp_path])
p.wait()
os.remove(tmp_path)
def config_dict(config_path=CONFIG_PATH):
"""Returns the config as dictionary,
where the elements have intuitively correct types.
"""
config = configparser.ConfigParser()
config.read(config_path)
d = dict()
for section_key in config.sections():
sd = dict()
section = config[section_key]
for key in section:
val = section[key]
try:
sd[key] = int(val)
except ValueError:
try:
sd[key] = float(val)
except ValueError:
try:
sd[key] = section.getboolean(key)
except ValueError:
sd[key] = val
d[section_key] = sd
return d
def convert_input_strings(config_dct, dirs):
if 'manual_decay_iters' in config_dct and 'manual_decay_lrs' in config_dct:
iters_lst = config_dct['manual_decay_iters'].split(',')
lrs_lst = config_dct['manual_decay_lrs'].split(',')
iters_lst = [int(i) for i in iters_lst]
lrs_lst = [float(l) for l in lrs_lst]
config_dct['manual_decay_iters'] = iters_lst
config_dct['manual_decay_lrs'] = lrs_lst
config_dct['num_iters'] = sum(iters_lst)
if 'finetune' in config_dct:
finetune = []
for name in config_dct['finetune'].split(","):
ckpt_dir = os.path.join(dirs['checkpoints'], name)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
if ckpt is None:
ckpt_dir = os.path.join(dirs['log'], 'ex', name)
ckpt = tf.train.get_checkpoint_state(ckpt_dir)
assert ckpt, "Could not load experiment " + name
finetune.append(ckpt)
config_dct['finetune'] = finetune
def tryremove(name, file=False):
try:
if file:
os.remove(name)
else:
rmtree(name)
except OSError:
pass
|