File size: 4,313 Bytes
d2a9154 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 | """
Train and evaluate the baselines and ablations (run from the repository root).
python -m experiments.run_all --steps eval_released
python -m experiments.run_all --steps train,eval_runs --variants s2f_full,unet,unet_gan
steps:
eval_released released checkpoints and input-free baseline -> experiments/results/eval/
train train the listed variants, one job per GPU (--gpus); finished runs are skipped
eval_runs evaluate every experiments/runs/*/best_checkpoint.pth
"""
import argparse
import json
import os
import subprocess
import sys
import time
from experiments.configs import PRIORITY, get_config
from experiments.layout import RUNS_DIR, S2F_ROOT
def sh(args):
print("\n$ " + " ".join(args), flush=True)
r = subprocess.run([sys.executable, "-m"] + args, cwd=S2F_ROOT)
if r.returncode:
sys.exit(r.returncode)
def finished(variant, seed):
h = os.path.join(RUNS_DIR, f"{variant}_seed{seed}", "history.json")
if not os.path.exists(h):
return False
return len(json.load(open(h))) >= get_config(variant)["epochs"]
def gpu_queue(queue, gpus, num_workers, extra=(), max_retries=2):
"""Run training jobs in parallel, one per GPU. A failed job is re-queued and resumes from its last checkpoint."""
running, failed, tries = {}, [], {}
queue = list(queue)
while queue or running:
for gpu in gpus:
if gpu not in running and queue:
v, seed = queue.pop(0)
run_dir = os.path.join(RUNS_DIR, f"{v}_seed{seed}")
os.makedirs(run_dir, exist_ok=True)
log = open(os.path.join(run_dir, "train.log"), "a")
env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=gpu, PYTHONUNBUFFERED="1")
cmd = [sys.executable, "-X", "faulthandler", "-m", "experiments.train", "--variant", v, "--seed", seed,
"--resume", "--num-workers", num_workers] + list(extra)
running[gpu] = (subprocess.Popen(cmd, cwd=S2F_ROOT, env=env, stdout=log, stderr=subprocess.STDOUT), v, seed, log)
print(f"[{time.strftime('%H:%M')}] GPU {gpu}: started {v} seed {seed} (log {run_dir}/train.log)", flush=True)
time.sleep(10 if extra else 30)
for gpu, (proc, v, seed, log) in list(running.items()):
if proc.poll() is not None:
log.close(); del running[gpu]
if proc.returncode == 0:
print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} finished", flush=True)
continue
tries[(v, seed)] = tries.get((v, seed), 0) + 1
if tries[(v, seed)] <= max_retries:
queue.insert(0, (v, seed))
print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}); "
f"resuming (retry {tries[(v, seed)]}/{max_retries})", flush=True)
else:
failed.append((v, seed))
print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}, see train.log)", flush=True)
if failed:
print(f"failed jobs: {failed}")
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--steps", default="eval_released")
ap.add_argument("--variants", default=",".join(PRIORITY))
ap.add_argument("--seeds", default="0")
ap.add_argument("--gpus", default="0,1")
ap.add_argument("--num-workers", default="2")
ap.add_argument("--extra-train-args", default="", help="passed to experiments.train, e.g. '--epochs 1 --max-batches 2'")
a = ap.parse_args()
steps = a.steps.split(",")
if "eval_released" in steps:
sh(["experiments.evaluate", "--released", "--meanmap"])
if "train" in steps:
queue = [(v, s) for v in a.variants.split(",") for s in a.seeds.split(",") if not finished(v, int(s))]
print(f"{len(queue)} training jobs on GPUs {a.gpus}: {queue}")
gpu_queue(queue, a.gpus.split(","), a.num_workers, a.extra_train_args.split())
if "eval_runs" in steps:
sh(["experiments.evaluate", "--runs"])
if __name__ == "__main__":
main()
|