| import os |
| import subprocess |
| import configparser |
| from shutil import rmtree |
| import tensorflow as tf |
|
|
|
|
| CONFIG_PATH = '../config.ini' |
| |
|
|
|
|
| def upload_gdrive(upload_dir, gdrive_filename): |
| |
| lst_lines = subprocess.Popen(['../scripts/gdrive', 'list'], |
| stdout=subprocess.PIPE) |
| existing_id = None |
| for line in lst_lines.stdout: |
| splits = line.split() |
| if str(splits[1], 'utf-8') == gdrive_filename: |
| existing_id = str(splits[0], 'utf-8') |
| tmp_path = os.path.join('/tmp', gdrive_filename) |
| if os.path.isfile(tmp_path): |
| os.remove(tmp_path) |
| p = subprocess.Popen(['/usr/bin/zip', '-r', tmp_path, upload_dir]) |
| p.wait() |
| if existing_id: |
| p = subprocess.Popen(['../scripts/gdrive', 'update', |
| existing_id, tmp_path]) |
| else: |
| p = subprocess.Popen(['../scripts/gdrive', 'upload', |
| '--name', gdrive_filename, |
| tmp_path]) |
| p.wait() |
| os.remove(tmp_path) |
|
|
|
|
| def config_dict(config_path=CONFIG_PATH): |
| """Returns the config as dictionary, |
| where the elements have intuitively correct types. |
| """ |
|
|
| config = configparser.ConfigParser() |
| config.read(config_path) |
|
|
| d = dict() |
| for section_key in config.sections(): |
| sd = dict() |
| section = config[section_key] |
| for key in section: |
| val = section[key] |
| try: |
| sd[key] = int(val) |
| except ValueError: |
| try: |
| sd[key] = float(val) |
| except ValueError: |
| try: |
| sd[key] = section.getboolean(key) |
| except ValueError: |
| sd[key] = val |
| d[section_key] = sd |
| return d |
|
|
|
|
| def convert_input_strings(config_dct, dirs): |
| if 'manual_decay_iters' in config_dct and 'manual_decay_lrs' in config_dct: |
| iters_lst = config_dct['manual_decay_iters'].split(',') |
| lrs_lst = config_dct['manual_decay_lrs'].split(',') |
| iters_lst = [int(i) for i in iters_lst] |
| lrs_lst = [float(l) for l in lrs_lst] |
| config_dct['manual_decay_iters'] = iters_lst |
| config_dct['manual_decay_lrs'] = lrs_lst |
| config_dct['num_iters'] = sum(iters_lst) |
|
|
| if 'finetune' in config_dct: |
| finetune = [] |
| for name in config_dct['finetune'].split(","): |
| ckpt_dir = os.path.join(dirs['checkpoints'], name) |
| ckpt = tf.train.get_checkpoint_state(ckpt_dir) |
| if ckpt is None: |
| ckpt_dir = os.path.join(dirs['log'], 'ex', name) |
| ckpt = tf.train.get_checkpoint_state(ckpt_dir) |
| assert ckpt, "Could not load experiment " + name |
| finetune.append(ckpt) |
| config_dct['finetune'] = finetune |
|
|
|
|
| def tryremove(name, file=False): |
| try: |
| if file: |
| os.remove(name) |
| else: |
| rmtree(name) |
| except OSError: |
| pass |
|
|
