File size: 4,313 Bytes
d2a9154
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
"""
Train and evaluate the baselines and ablations (run from the repository root).

    python -m experiments.run_all --steps eval_released
    python -m experiments.run_all --steps train,eval_runs --variants s2f_full,unet,unet_gan

steps:
  eval_released  released checkpoints and input-free baseline       -> experiments/results/eval/
  train          train the listed variants, one job per GPU (--gpus); finished runs are skipped
  eval_runs      evaluate every experiments/runs/*/best_checkpoint.pth
"""
import argparse
import json
import os
import subprocess
import sys
import time

from experiments.configs import PRIORITY, get_config
from experiments.layout import RUNS_DIR, S2F_ROOT


def sh(args):
    print("\n$ " + " ".join(args), flush=True)
    r = subprocess.run([sys.executable, "-m"] + args, cwd=S2F_ROOT)
    if r.returncode:
        sys.exit(r.returncode)


def finished(variant, seed):
    h = os.path.join(RUNS_DIR, f"{variant}_seed{seed}", "history.json")
    if not os.path.exists(h):
        return False
    return len(json.load(open(h))) >= get_config(variant)["epochs"]


def gpu_queue(queue, gpus, num_workers, extra=(), max_retries=2):
    """Run training jobs in parallel, one per GPU. A failed job is re-queued and resumes from its last checkpoint."""
    running, failed, tries = {}, [], {}
    queue = list(queue)
    while queue or running:
        for gpu in gpus:
            if gpu not in running and queue:
                v, seed = queue.pop(0)
                run_dir = os.path.join(RUNS_DIR, f"{v}_seed{seed}")
                os.makedirs(run_dir, exist_ok=True)
                log = open(os.path.join(run_dir, "train.log"), "a")
                env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=gpu, PYTHONUNBUFFERED="1")
                cmd = [sys.executable, "-X", "faulthandler", "-m", "experiments.train", "--variant", v, "--seed", seed,
                       "--resume", "--num-workers", num_workers] + list(extra)
                running[gpu] = (subprocess.Popen(cmd, cwd=S2F_ROOT, env=env, stdout=log, stderr=subprocess.STDOUT), v, seed, log)
                print(f"[{time.strftime('%H:%M')}] GPU {gpu}: started {v} seed {seed}  (log {run_dir}/train.log)", flush=True)
        time.sleep(10 if extra else 30)
        for gpu, (proc, v, seed, log) in list(running.items()):
            if proc.poll() is not None:
                log.close(); del running[gpu]
                if proc.returncode == 0:
                    print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} finished", flush=True)
                    continue
                tries[(v, seed)] = tries.get((v, seed), 0) + 1
                if tries[(v, seed)] <= max_retries:
                    queue.insert(0, (v, seed))
                    print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}); "
                          f"resuming (retry {tries[(v, seed)]}/{max_retries})", flush=True)
                else:
                    failed.append((v, seed))
                    print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}, see train.log)", flush=True)
    if failed:
        print(f"failed jobs: {failed}")


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--steps", default="eval_released")
    ap.add_argument("--variants", default=",".join(PRIORITY))
    ap.add_argument("--seeds", default="0")
    ap.add_argument("--gpus", default="0,1")
    ap.add_argument("--num-workers", default="2")
    ap.add_argument("--extra-train-args", default="", help="passed to experiments.train, e.g. '--epochs 1 --max-batches 2'")
    a = ap.parse_args()
    steps = a.steps.split(",")
    if "eval_released" in steps:
        sh(["experiments.evaluate", "--released", "--meanmap"])
    if "train" in steps:
        queue = [(v, s) for v in a.variants.split(",") for s in a.seeds.split(",") if not finished(v, int(s))]
        print(f"{len(queue)} training jobs on GPUs {a.gpus}: {queue}")
        gpu_queue(queue, a.gpus.split(","), a.num_workers, a.extra_train_args.split())
    if "eval_runs" in steps:
        sh(["experiments.evaluate", "--runs"])


if __name__ == "__main__":
    main()