anhtld commited on
Commit
6ede549
·
verified ·
1 Parent(s): 8be4a18

Auto-sync: 2026-06-27 11:04:40 (part 2)

Browse files
scripts/slurm/summarize_h16_policy_ckpt.sbatch CHANGED
@@ -12,6 +12,8 @@ set -euo pipefail
12
  PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
13
  RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
14
  OBJECTIVE="${OBJECTIVE:-base}"
 
 
15
  if [[ -z "${PYTHON:-}" ]]; then
16
  if [[ -x "$PROJECT_DIR/.venv/bin/python" ]]; then
17
  PYTHON="$PROJECT_DIR/.venv/bin/python"
@@ -21,6 +23,8 @@ if [[ -z "${PYTHON:-}" ]]; then
21
  fi
22
  export RUN_ROOT
23
  export OBJECTIVE
 
 
24
 
25
  cd "$PROJECT_DIR"
26
  mkdir -p results outputs/hpc/logs
@@ -35,12 +39,14 @@ from pathlib import Path
35
 
36
  run_root = Path(os.environ["RUN_ROOT"])
37
  objective = os.environ.get("OBJECTIVE", "base")
 
 
38
  base_dir = run_root / objective
39
  baseline_h4 = 0.2967
40
  baseline_h16_rank = 0.29739130434782607
41
 
42
  rows = []
43
- for result_path in sorted(base_dir.glob("seed_*/policy_rollout.json")):
44
  data = json.loads(result_path.read_text())
45
  seed = int(result_path.parent.name.split("_")[-1])
46
  metrics_path = result_path.parent / "metrics.json"
@@ -52,6 +58,9 @@ for result_path in sorted(base_dir.glob("seed_*/policy_rollout.json")):
52
  "seed": seed,
53
  "path": str(result_path),
54
  "num_groups": data.get("num_groups"),
 
 
 
55
  "policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
56
  "policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
57
  "oracle_success_rate": data.get("oracle_success_rate", 0.0),
@@ -67,6 +76,7 @@ mses = [row["action_mse_to_best"] for row in rows]
67
  summary = {
68
  "run_root": str(run_root),
69
  "objective": objective,
 
70
  "num_completed": len(rows),
71
  "baseline_h4_policy_success": baseline_h4,
72
  "baseline_h16_rank_checkpoint_success": baseline_h16_rank,
@@ -83,7 +93,16 @@ summary = {
83
  "rows": rows,
84
  }
85
 
86
- summary_stem = "policy_ckpt" if objective == "base" else f"policy_ckpt_{objective}"
 
 
 
 
 
 
 
 
 
87
  json_path = Path(f"results/h16_{summary_stem}_summary.json")
88
  json_path.write_text(json.dumps(summary, indent=2))
89
 
@@ -91,6 +110,8 @@ lines = [
91
  "# h=16 Best-Policy Checkpoint Rollout",
92
  "",
93
  f"Run root: `{run_root}`",
 
 
94
  f"Completed seeds: {len(rows)}",
95
  f"Baseline h=4 policy success: {baseline_h4:.2%}",
96
  f"Baseline h=16 rank-checkpoint success: {baseline_h16_rank:.2%}",
@@ -100,13 +121,17 @@ lines = [
100
  f"Mean progress: {summary['mean_progress']:.2%}",
101
  f"Mean action MSE to best: {summary['mean_action_mse_to_best']:.3f}",
102
  "",
103
- "| seed | success | progress | oracle | action MSE |",
104
- "|---:|---:|---:|---:|---:|",
105
  ]
106
  for row in rows:
107
  lines.append(
108
- "| {seed} | {success:.2%} | {progress:.2%} | {oracle:.2%} | {mse:.3f} |".format(
 
109
  seed=row["seed"],
 
 
 
110
  success=row["policy_rollout_success_rate"],
111
  progress=row["policy_rollout_progress"],
112
  oracle=row["oracle_success_rate"],
 
12
  PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
13
  RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_policy_ckpt_runs}"
14
  OBJECTIVE="${OBJECTIVE:-base}"
15
+ OUT_NAME="${OUT_NAME:-policy_rollout.json}"
16
+ SUMMARY_TAG="${SUMMARY_TAG:-}"
17
  if [[ -z "${PYTHON:-}" ]]; then
18
  if [[ -x "$PROJECT_DIR/.venv/bin/python" ]]; then
19
  PYTHON="$PROJECT_DIR/.venv/bin/python"
 
23
  fi
24
  export RUN_ROOT
25
  export OBJECTIVE
26
+ export OUT_NAME
27
+ export SUMMARY_TAG
28
 
29
  cd "$PROJECT_DIR"
30
  mkdir -p results outputs/hpc/logs
 
39
 
40
  run_root = Path(os.environ["RUN_ROOT"])
41
  objective = os.environ.get("OBJECTIVE", "base")
42
+ out_name = os.environ.get("OUT_NAME", "policy_rollout.json")
43
+ summary_tag = os.environ.get("SUMMARY_TAG", "")
44
  base_dir = run_root / objective
45
  baseline_h4 = 0.2967
46
  baseline_h16_rank = 0.29739130434782607
47
 
48
  rows = []
49
+ for result_path in sorted(base_dir.glob(f"seed_*/{out_name}")):
50
  data = json.loads(result_path.read_text())
51
  seed = int(result_path.parent.name.split("_")[-1])
52
  metrics_path = result_path.parent / "metrics.json"
 
58
  "seed": seed,
59
  "path": str(result_path),
60
  "num_groups": data.get("num_groups"),
61
+ "selection_mode": data.get("selection_mode"),
62
+ "num_candidates": data.get("num_candidates"),
63
+ "candidate_sigma": data.get("candidate_sigma"),
64
  "policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
65
  "policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
66
  "oracle_success_rate": data.get("oracle_success_rate", 0.0),
 
76
  summary = {
77
  "run_root": str(run_root),
78
  "objective": objective,
79
+ "out_name": out_name,
80
  "num_completed": len(rows),
81
  "baseline_h4_policy_success": baseline_h4,
82
  "baseline_h16_rank_checkpoint_success": baseline_h16_rank,
 
93
  "rows": rows,
94
  }
95
 
96
+ out_stem = out_name[:-5] if out_name.endswith(".json") else out_name
97
+ out_stem = out_stem.replace("_rollout", "")
98
+ if summary_tag:
99
+ summary_stem = summary_tag
100
+ elif objective == "base" and out_stem == "policy":
101
+ summary_stem = "policy_ckpt"
102
+ elif out_stem == "policy":
103
+ summary_stem = f"policy_ckpt_{objective}"
104
+ else:
105
+ summary_stem = f"policy_ckpt_{objective}_{out_stem}"
106
  json_path = Path(f"results/h16_{summary_stem}_summary.json")
107
  json_path.write_text(json.dumps(summary, indent=2))
108
 
 
110
  "# h=16 Best-Policy Checkpoint Rollout",
111
  "",
112
  f"Run root: `{run_root}`",
113
+ f"Objective: `{objective}`",
114
+ f"Result file: `{out_name}`",
115
  f"Completed seeds: {len(rows)}",
116
  f"Baseline h=4 policy success: {baseline_h4:.2%}",
117
  f"Baseline h=16 rank-checkpoint success: {baseline_h16_rank:.2%}",
 
121
  f"Mean progress: {summary['mean_progress']:.2%}",
122
  f"Mean action MSE to best: {summary['mean_action_mse_to_best']:.3f}",
123
  "",
124
+ "| seed | mode | k | sigma | success | progress | oracle | action MSE |",
125
+ "|---:|---|---:|---:|---:|---:|---:|---:|",
126
  ]
127
  for row in rows:
128
  lines.append(
129
+ "| {seed} | {mode} | {k} | {sigma:.2f} | {success:.2%} | "
130
+ "{progress:.2%} | {oracle:.2%} | {mse:.3f} |".format(
131
  seed=row["seed"],
132
+ mode=row.get("selection_mode") or "policy",
133
+ k=row.get("num_candidates") or 1,
134
+ sigma=row.get("candidate_sigma") or 0.0,
135
  success=row["policy_rollout_success_rate"],
136
  progress=row["policy_rollout_progress"],
137
  oracle=row["oracle_success_rate"],