Spaces:
Sleeping
Sleeping
File size: 2,690 Bytes
bd096d2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | # -*- coding: utf-8 -*-
# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
# holder of all proprietary rights on this computer program.
# You can only use this computer program if you have closed
# a license agreement with MPG or you get the right to use the computer
# program from someone who is authorized to grant you that right.
# Any use of the computer program without a valid license is prohibited and
# liable to prosecution.
#
# Copyright©2023 Max-Planck-Gesellschaft zur Förderung
# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
# for Intelligent Systems. All rights reserved.
#
# Contact: mica@tue.mpg.de
import os
import time
from glob import glob
from shutil import copyfile
logs = '/home/wzielonka/projects/MICA/testing/now/logs/'
jobs = '/home/wzielonka/projects/MICA/testing/now/jobs/'
root = '/home/wzielonka/projects/MICA/output/'
experiments = []
def test():
global experiments
if len(experiments) == 0:
experiments = list(filter(lambda f: 'condor' not in f, os.listdir('../../output/')))
os.system('rm -rf logs')
os.system('rm -rf jobs')
os.makedirs('logs', exist_ok=True)
os.makedirs('jobs', exist_ok=True)
for experiment in sorted(experiments):
print(f'Testing {experiment}')
copyfile(f'{root}{experiment}/model.tar', f'{root}{experiment}/best_models/best_model_last.tar')
for idx, checkpoint in enumerate(glob(root + experiment + f'/best_models/*.tar')):
model_name = checkpoint.split('/')[-1].split('.')[0]
model_name = model_name.replace('best_model_', 'now_test_')
predicted_meshes = f'{root}{experiment}/{model_name}/predicted_meshes/'
run = f'{experiment}_{str(idx).zfill(5)}'
with open(f'{jobs}/{run}.sub', 'w') as fid:
fid.write('executable = /bin/bash\n')
arguments = f'/home/wzielonka/projects/MICA/testing/now/template.sh {experiment} {checkpoint} now {predicted_meshes}'
fid.write(f'arguments = {arguments}\n')
fid.write(f'error = {logs}{run}.err\n')
fid.write(f'output = {logs}{run}.out\n')
fid.write(f'log = {logs}{run}.log\n')
fid.write(f'request_cpus = 4\n')
fid.write(f'request_gpus = 1\n')
fid.write(f'requirements = (TARGET.CUDAGlobalMemoryMb > 5000) && (TARGET.CUDAGlobalMemoryMb < 33000)\n')
fid.write(f'request_memory = 8192\n')
fid.write(f'queue\n')
os.system(f'condor_submit_bid 512 {jobs}/{run}.sub')
time.sleep(2)
if __name__ == '__main__':
test()
|