Spaces:
Sleeping
Sleeping
| # -*- coding: utf-8 -*- | |
| # Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is | |
| # holder of all proprietary rights on this computer program. | |
| # You can only use this computer program if you have closed | |
| # a license agreement with MPG or you get the right to use the computer | |
| # program from someone who is authorized to grant you that right. | |
| # Any use of the computer program without a valid license is prohibited and | |
| # liable to prosecution. | |
| # | |
| # Copyright©2023 Max-Planck-Gesellschaft zur Förderung | |
| # der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute | |
| # for Intelligent Systems. All rights reserved. | |
| # | |
| # Contact: mica@tue.mpg.de | |
| import os | |
| import time | |
| from glob import glob | |
| from shutil import copyfile | |
| logs = '/home/wzielonka/projects/MICA/testing/now/logs/' | |
| jobs = '/home/wzielonka/projects/MICA/testing/now/jobs/' | |
| root = '/home/wzielonka/projects/MICA/output/' | |
| experiments = [] | |
| def test(): | |
| global experiments | |
| if len(experiments) == 0: | |
| experiments = list(filter(lambda f: 'condor' not in f, os.listdir('../../output/'))) | |
| os.system('rm -rf logs') | |
| os.system('rm -rf jobs') | |
| os.makedirs('logs', exist_ok=True) | |
| os.makedirs('jobs', exist_ok=True) | |
| for experiment in sorted(experiments): | |
| print(f'Testing {experiment}') | |
| copyfile(f'{root}{experiment}/model.tar', f'{root}{experiment}/best_models/best_model_last.tar') | |
| for idx, checkpoint in enumerate(glob(root + experiment + f'/best_models/*.tar')): | |
| model_name = checkpoint.split('/')[-1].split('.')[0] | |
| model_name = model_name.replace('best_model_', 'now_test_') | |
| predicted_meshes = f'{root}{experiment}/{model_name}/predicted_meshes/' | |
| run = f'{experiment}_{str(idx).zfill(5)}' | |
| with open(f'{jobs}/{run}.sub', 'w') as fid: | |
| fid.write('executable = /bin/bash\n') | |
| arguments = f'/home/wzielonka/projects/MICA/testing/now/template.sh {experiment} {checkpoint} now {predicted_meshes}' | |
| fid.write(f'arguments = {arguments}\n') | |
| fid.write(f'error = {logs}{run}.err\n') | |
| fid.write(f'output = {logs}{run}.out\n') | |
| fid.write(f'log = {logs}{run}.log\n') | |
| fid.write(f'request_cpus = 4\n') | |
| fid.write(f'request_gpus = 1\n') | |
| fid.write(f'requirements = (TARGET.CUDAGlobalMemoryMb > 5000) && (TARGET.CUDAGlobalMemoryMb < 33000)\n') | |
| fid.write(f'request_memory = 8192\n') | |
| fid.write(f'queue\n') | |
| os.system(f'condor_submit_bid 512 {jobs}/{run}.sub') | |
| time.sleep(2) | |
| if __name__ == '__main__': | |
| test() | |