Auto-sync: 2026-06-27 11:04:40 (part 2)
Browse files
scripts/slurm/summarize_h16_policy_ckpt.sbatch
CHANGED
|
@@ -12,6 +12,8 @@ set -euo pipefail
|
|
| 12 |
PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
|
| 13 |
RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
|
| 14 |
OBJECTIVE="${OBJECTIVE:-base}"
|
|
|
|
|
|
|
| 15 |
if [[ -z "${PYTHON:-}" ]]; then
|
| 16 |
if [[ -x "$PROJECT_DIR/.venv/bin/python" ]]; then
|
| 17 |
PYTHON="$PROJECT_DIR/.venv/bin/python"
|
|
@@ -21,6 +23,8 @@ if [[ -z "${PYTHON:-}" ]]; then
|
|
| 21 |
fi
|
| 22 |
export RUN_ROOT
|
| 23 |
export OBJECTIVE
|
|
|
|
|
|
|
| 24 |
|
| 25 |
cd "$PROJECT_DIR"
|
| 26 |
mkdir -p results outputs/hpc/logs
|
|
@@ -35,12 +39,14 @@ from pathlib import Path
|
|
| 35 |
|
| 36 |
run_root = Path(os.environ["RUN_ROOT"])
|
| 37 |
objective = os.environ.get("OBJECTIVE", "base")
|
|
|
|
|
|
|
| 38 |
base_dir = run_root / objective
|
| 39 |
baseline_h4 = 0.2967
|
| 40 |
baseline_h16_rank = 0.29739130434782607
|
| 41 |
|
| 42 |
rows = []
|
| 43 |
-
for result_path in sorted(base_dir.glob("seed_*/
|
| 44 |
data = json.loads(result_path.read_text())
|
| 45 |
seed = int(result_path.parent.name.split("_")[-1])
|
| 46 |
metrics_path = result_path.parent / "metrics.json"
|
|
@@ -52,6 +58,9 @@ for result_path in sorted(base_dir.glob("seed_*/policy_rollout.json")):
|
|
| 52 |
"seed": seed,
|
| 53 |
"path": str(result_path),
|
| 54 |
"num_groups": data.get("num_groups"),
|
|
|
|
|
|
|
|
|
|
| 55 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 56 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
| 57 |
"oracle_success_rate": data.get("oracle_success_rate", 0.0),
|
|
@@ -67,6 +76,7 @@ mses = [row["action_mse_to_best"] for row in rows]
|
|
| 67 |
summary = {
|
| 68 |
"run_root": str(run_root),
|
| 69 |
"objective": objective,
|
|
|
|
| 70 |
"num_completed": len(rows),
|
| 71 |
"baseline_h4_policy_success": baseline_h4,
|
| 72 |
"baseline_h16_rank_checkpoint_success": baseline_h16_rank,
|
|
@@ -83,7 +93,16 @@ summary = {
|
|
| 83 |
"rows": rows,
|
| 84 |
}
|
| 85 |
|
| 86 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
json_path = Path(f"results/h16_{summary_stem}_summary.json")
|
| 88 |
json_path.write_text(json.dumps(summary, indent=2))
|
| 89 |
|
|
@@ -91,6 +110,8 @@ lines = [
|
|
| 91 |
"# h=16 Best-Policy Checkpoint Rollout",
|
| 92 |
"",
|
| 93 |
f"Run root: `{run_root}`",
|
|
|
|
|
|
|
| 94 |
f"Completed seeds: {len(rows)}",
|
| 95 |
f"Baseline h=4 policy success: {baseline_h4:.2%}",
|
| 96 |
f"Baseline h=16 rank-checkpoint success: {baseline_h16_rank:.2%}",
|
|
@@ -100,13 +121,17 @@ lines = [
|
|
| 100 |
f"Mean progress: {summary['mean_progress']:.2%}",
|
| 101 |
f"Mean action MSE to best: {summary['mean_action_mse_to_best']:.3f}",
|
| 102 |
"",
|
| 103 |
-
"| seed | success | progress | oracle | action MSE |",
|
| 104 |
-
"|---:|---:|---:|---:|---:|",
|
| 105 |
]
|
| 106 |
for row in rows:
|
| 107 |
lines.append(
|
| 108 |
-
"| {seed} | {
|
|
|
|
| 109 |
seed=row["seed"],
|
|
|
|
|
|
|
|
|
|
| 110 |
success=row["policy_rollout_success_rate"],
|
| 111 |
progress=row["policy_rollout_progress"],
|
| 112 |
oracle=row["oracle_success_rate"],
|
|
|
|
| 12 |
PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
|
| 13 |
RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
|
| 14 |
OBJECTIVE="${OBJECTIVE:-base}"
|
| 15 |
+
OUT_NAME="${OUT_NAME:-policy_rollout.json}"
|
| 16 |
+
SUMMARY_TAG="${SUMMARY_TAG:-}"
|
| 17 |
if [[ -z "${PYTHON:-}" ]]; then
|
| 18 |
if [[ -x "$PROJECT_DIR/.venv/bin/python" ]]; then
|
| 19 |
PYTHON="$PROJECT_DIR/.venv/bin/python"
|
|
|
|
| 23 |
fi
|
| 24 |
export RUN_ROOT
|
| 25 |
export OBJECTIVE
|
| 26 |
+
export OUT_NAME
|
| 27 |
+
export SUMMARY_TAG
|
| 28 |
|
| 29 |
cd "$PROJECT_DIR"
|
| 30 |
mkdir -p results outputs/hpc/logs
|
|
|
|
| 39 |
|
| 40 |
run_root = Path(os.environ["RUN_ROOT"])
|
| 41 |
objective = os.environ.get("OBJECTIVE", "base")
|
| 42 |
+
out_name = os.environ.get("OUT_NAME", "policy_rollout.json")
|
| 43 |
+
summary_tag = os.environ.get("SUMMARY_TAG", "")
|
| 44 |
base_dir = run_root / objective
|
| 45 |
baseline_h4 = 0.2967
|
| 46 |
baseline_h16_rank = 0.29739130434782607
|
| 47 |
|
| 48 |
rows = []
|
| 49 |
+
for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
|
| 50 |
data = json.loads(result_path.read_text())
|
| 51 |
seed = int(result_path.parent.name.split("_")[-1])
|
| 52 |
metrics_path = result_path.parent / "metrics.json"
|
|
|
|
| 58 |
"seed": seed,
|
| 59 |
"path": str(result_path),
|
| 60 |
"num_groups": data.get("num_groups"),
|
| 61 |
+
"selection_mode": data.get("selection_mode"),
|
| 62 |
+
"num_candidates": data.get("num_candidates"),
|
| 63 |
+
"candidate_sigma": data.get("candidate_sigma"),
|
| 64 |
"policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
|
| 65 |
"policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
|
| 66 |
"oracle_success_rate": data.get("oracle_success_rate", 0.0),
|
|
|
|
| 76 |
summary = {
|
| 77 |
"run_root": str(run_root),
|
| 78 |
"objective": objective,
|
| 79 |
+
"out_name": out_name,
|
| 80 |
"num_completed": len(rows),
|
| 81 |
"baseline_h4_policy_success": baseline_h4,
|
| 82 |
"baseline_h16_rank_checkpoint_success": baseline_h16_rank,
|
|
|
|
| 93 |
"rows": rows,
|
| 94 |
}
|
| 95 |
|
| 96 |
+
out_stem = out_name[:-5] if out_name.endswith(".json") else out_name
|
| 97 |
+
out_stem = out_stem.replace("_rollout", "")
|
| 98 |
+
if summary_tag:
|
| 99 |
+
summary_stem = summary_tag
|
| 100 |
+
elif objective == "base" and out_stem == "policy":
|
| 101 |
+
summary_stem = "policy_ckpt"
|
| 102 |
+
elif out_stem == "policy":
|
| 103 |
+
summary_stem = f"policy_ckpt_{objective}"
|
| 104 |
+
else:
|
| 105 |
+
summary_stem = f"policy_ckpt_{objective}_{out_stem}"
|
| 106 |
json_path = Path(f"results/h16_{summary_stem}_summary.json")
|
| 107 |
json_path.write_text(json.dumps(summary, indent=2))
|
| 108 |
|
|
|
|
| 110 |
"# h=16 Best-Policy Checkpoint Rollout",
|
| 111 |
"",
|
| 112 |
f"Run root: `{run_root}`",
|
| 113 |
+
f"Objective: `{objective}`",
|
| 114 |
+
f"Result file: `{out_name}`",
|
| 115 |
f"Completed seeds: {len(rows)}",
|
| 116 |
f"Baseline h=4 policy success: {baseline_h4:.2%}",
|
| 117 |
f"Baseline h=16 rank-checkpoint success: {baseline_h16_rank:.2%}",
|
|
|
|
| 121 |
f"Mean progress: {summary['mean_progress']:.2%}",
|
| 122 |
f"Mean action MSE to best: {summary['mean_action_mse_to_best']:.3f}",
|
| 123 |
"",
|
| 124 |
+
"| seed | mode | k | sigma | success | progress | oracle | action MSE |",
|
| 125 |
+
"|---:|---|---:|---:|---:|---:|---:|---:|",
|
| 126 |
]
|
| 127 |
for row in rows:
|
| 128 |
lines.append(
|
| 129 |
+
"| {seed} | {mode} | {k} | {sigma:.2f} | {success:.2%} | "
|
| 130 |
+
"{progress:.2%} | {oracle:.2%} | {mse:.3f} |".format(
|
| 131 |
seed=row["seed"],
|
| 132 |
+
mode=row.get("selection_mode") or "policy",
|
| 133 |
+
k=row.get("num_candidates") or 1,
|
| 134 |
+
sigma=row.get("candidate_sigma") or 0.0,
|
| 135 |
success=row["policy_rollout_success_rate"],
|
| 136 |
progress=row["policy_rollout_progress"],
|
| 137 |
oracle=row["oracle_success_rate"],
|