Download experiments/run_all.py from Angione-Lab/Shape2Force: direct link, hf CLI and curl.
- Browser
- Download file 4.31 kB
-
https://huggingface.co/Angione-Lab/Shape2Force/resolve/main/experiments/run_all.py
- Command line
-
hf download hf://Angione-Lab/Shape2Force/experiments/run_all.py
-
curl -L -o run_all.py https://huggingface.co/Angione-Lab/Shape2Force/resolve/main/experiments/run_all.py
4.31 kB
| """ | |
| Train and evaluate the baselines and ablations (run from the repository root). | |
| python -m experiments.run_all --steps eval_released | |
| python -m experiments.run_all --steps train,eval_runs --variants s2f_full,unet,unet_gan | |
| steps: | |
| eval_released released checkpoints and input-free baseline -> experiments/results/eval/ | |
| train train the listed variants, one job per GPU (--gpus); finished runs are skipped | |
| eval_runs evaluate every experiments/runs/*/best_checkpoint.pth | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import subprocess | |
| import sys | |
| import time | |
| from experiments.configs import PRIORITY, get_config | |
| from experiments.layout import RUNS_DIR, S2F_ROOT | |
| def sh(args): | |
| print("\n$ " + " ".join(args), flush=True) | |
| r = subprocess.run([sys.executable, "-m"] + args, cwd=S2F_ROOT) | |
| if r.returncode: | |
| sys.exit(r.returncode) | |
| def finished(variant, seed): | |
| h = os.path.join(RUNS_DIR, f"{variant}_seed{seed}", "history.json") | |
| if not os.path.exists(h): | |
| return False | |
| return len(json.load(open(h))) >= get_config(variant)["epochs"] | |
| def gpu_queue(queue, gpus, num_workers, extra=(), max_retries=2): | |
| """Run training jobs in parallel, one per GPU. A failed job is re-queued and resumes from its last checkpoint.""" | |
| running, failed, tries = {}, [], {} | |
| queue = list(queue) | |
| while queue or running: | |
| for gpu in gpus: | |
| if gpu not in running and queue: | |
| v, seed = queue.pop(0) | |
| run_dir = os.path.join(RUNS_DIR, f"{v}_seed{seed}") | |
| os.makedirs(run_dir, exist_ok=True) | |
| log = open(os.path.join(run_dir, "train.log"), "a") | |
| env = dict(os.environ, CUDA_DEVICE_ORDER="PCI_BUS_ID", CUDA_VISIBLE_DEVICES=gpu, PYTHONUNBUFFERED="1") | |
| cmd = [sys.executable, "-X", "faulthandler", "-m", "experiments.train", "--variant", v, "--seed", seed, | |
| "--resume", "--num-workers", num_workers] + list(extra) | |
| running[gpu] = (subprocess.Popen(cmd, cwd=S2F_ROOT, env=env, stdout=log, stderr=subprocess.STDOUT), v, seed, log) | |
| print(f"[{time.strftime('%H:%M')}] GPU {gpu}: started {v} seed {seed} (log {run_dir}/train.log)", flush=True) | |
| time.sleep(10 if extra else 30) | |
| for gpu, (proc, v, seed, log) in list(running.items()): | |
| if proc.poll() is not None: | |
| log.close(); del running[gpu] | |
| if proc.returncode == 0: | |
| print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} finished", flush=True) | |
| continue | |
| tries[(v, seed)] = tries.get((v, seed), 0) + 1 | |
| if tries[(v, seed)] <= max_retries: | |
| queue.insert(0, (v, seed)) | |
| print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}); " | |
| f"resuming (retry {tries[(v, seed)]}/{max_retries})", flush=True) | |
| else: | |
| failed.append((v, seed)) | |
| print(f"[{time.strftime('%H:%M')}] GPU {gpu}: {v} seed {seed} failed (exit {proc.returncode}, see train.log)", flush=True) | |
| if failed: | |
| print(f"failed jobs: {failed}") | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--steps", default="eval_released") | |
| ap.add_argument("--variants", default=",".join(PRIORITY)) | |
| ap.add_argument("--seeds", default="0") | |
| ap.add_argument("--gpus", default="0,1") | |
| ap.add_argument("--num-workers", default="2") | |
| ap.add_argument("--extra-train-args", default="", help="passed to experiments.train, e.g. '--epochs 1 --max-batches 2'") | |
| a = ap.parse_args() | |
| steps = a.steps.split(",") | |
| if "eval_released" in steps: | |
| sh(["experiments.evaluate", "--released", "--meanmap"]) | |
| if "train" in steps: | |
| queue = [(v, s) for v in a.variants.split(",") for s in a.seeds.split(",") if not finished(v, int(s))] | |
| print(f"{len(queue)} training jobs on GPUs {a.gpus}: {queue}") | |
| gpu_queue(queue, a.gpus.split(","), a.num_workers, a.extra_train_args.split()) | |
| if "eval_runs" in steps: | |
| sh(["experiments.evaluate", "--runs"]) | |
| if __name__ == "__main__": | |
| main() | |