Auto-sync: 2026-06-27 09:54:24 (part 2)

scripts/slurm/eval_h16_field_sweep.sbatch

@@ -0,0 +1,93 @@
+#!/bin/bash
+#SBATCH --job-name=eval_h16_field
+#SBATCH --account=def-yalda_gpu
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --gres=gpu:nvidia_h100_80gb_hbm3_1g.10gb:1
+#SBATCH --mem=32G
+#SBATCH --time=03:00:00
+#SBATCH --output=outputs/hpc/logs/%x_%A_%a.out
+#SBATCH --error=outputs/hpc/logs/%x_%A_%a.err
+#SBATCH --array=0-11
+
+set -euo pipefail
+
+# Field-guided h=16 rollout sweep.
+# Each job evaluates one seed/config pair by sampling model-generated action chunks,
+# scoring them with DoVLA's learned interventional field, and executing only the best.
+
+PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
+SCRATCH_ROOT="/scratch/$USER/dovla"
+SIF="${SIF:-$SCRATCH_ROOT/containers/pytorch_2.7.1_cuda12.8.sif}"
+PYTHON="${PYTHON:-$SCRATCH_ROOT/envs/maniskill/bin/python}"
+NATIVE_LIBS="$SCRATCH_ROOT/native_libs/lib"
+CPU_RENDER_LIBS="$SCRATCH_ROOT/cpu_render_libs"
+CA_BUNDLE="$SCRATCH_ROOT/ca-bundle.crt"
+VULKAN_ICD="$CPU_RENDER_LIBS/share/vulkan/icd.d/lvp_icd.x86_64.json"
+RUNTIME_DIR="/tmp/$USER/dovla-field-rollout-$SLURM_JOB_ID-${SLURM_ARRAY_TASK_ID:-0}"
+CACHE_DIR="/tmp/$USER/dovla-field-mesa-$SLURM_JOB_ID-${SLURM_ARRAY_TASK_ID:-0}"
+
+SEEDS=(0 1 2)
+NUM_CANDIDATES_GRID=(8 16 32 64)
+CANDIDATE_SIGMA_GRID=(0.10 0.20 0.35 0.50)
+
+CONFIG_IDX=$((SLURM_ARRAY_TASK_ID / ${#SEEDS[@]}))
+SEED_IDX=$((SLURM_ARRAY_TASK_ID % ${#SEEDS[@]}))
+SEED="${SEEDS[$SEED_IDX]}"
+NUM_CANDIDATES="${NUM_CANDIDATES_GRID[$CONFIG_IDX]}"
+CANDIDATE_SIGMA="${CANDIDATE_SIGMA_GRID[$CONFIG_IDX]}"
+SELECTION_SEED=$((91000 + CONFIG_IDX * 1000 + SEED))
+
+DATASET="${DATASET:-$SCRATCH_ROOT/experiments/six_task_h16_collection}"
+CHECKPOINT="${CHECKPOINT:-$SCRATCH_ROOT/experiments/dovla_h16_rollout_runs/seed_$SEED/best.pt}"
+RUN_ROOT="${RUN_ROOT:-$SCRATCH_ROOT/experiments/dovla_h16_field_sweep}"
+OUT_DIR="$RUN_ROOT/k${NUM_CANDIDATES}_sigma${CANDIDATE_SIGMA}/seed_${SEED}"
+OUT="$OUT_DIR/online_rollout.json"
+MAX_GROUPS="${MAX_GROUPS:-700}"
+GROUP_BATCH_SIZE="${GROUP_BATCH_SIZE:-16}"
+
+module load StdEnv/2023 apptainer/1.4.5
+cd "$PROJECT_DIR"
+mkdir -p outputs/hpc/logs "$RUNTIME_DIR" "$CACHE_DIR" "$OUT_DIR"
+chmod 700 "$RUNTIME_DIR"
+
+export OMP_NUM_THREADS=1
+export OPENBLAS_NUM_THREADS=1
+export MKL_NUM_THREADS=1
+export DOVLA_TORCH_THREADS=1
+
+ENVS="LD_LIBRARY_PATH=$CPU_RENDER_LIBS/lib:$NATIVE_LIBS:/.singularity.d/libs,VK_ICD_FILENAMES=$VULKAN_ICD,VK_DRIVER_FILES=$VULKAN_ICD,XDG_RUNTIME_DIR=$RUNTIME_DIR,MESA_SHADER_CACHE_DIR=$CACHE_DIR,LIBGL_ALWAYS_SOFTWARE=1,LP_NUM_THREADS=1,SSL_CERT_FILE=$CA_BUNDLE,REQUESTS_CA_BUNDLE=$CA_BUNDLE,OMP_NUM_THREADS=1,OPENBLAS_NUM_THREADS=1,MKL_NUM_THREADS=1,DOVLA_TORCH_THREADS=1,MPLBACKEND=Agg,PYTHONDONTWRITEBYTECODE=1"
+
+echo "=================================================="
+echo "Online Rollout Evaluation - h=16 Field-Guided"
+echo "Array task: $SLURM_ARRAY_TASK_ID"
+echo "Seed: $SEED"
+echo "Candidates: $NUM_CANDIDATES"
+echo "Candidate sigma: $CANDIDATE_SIGMA"
+echo "Selection seed: $SELECTION_SEED"
+echo "Checkpoint: $CHECKPOINT"
+echo "Dataset: $DATASET"
+echo "Out: $OUT"
+echo "=================================================="
+
+apptainer exec --nv --env "$ENVS" \
+  -B "$PROJECT_DIR:$PROJECT_DIR" \
+  -B "/scratch/$USER:/scratch/$USER" \
+  "$SIF" "$PYTHON" scripts/eval_maniskill_policy_rollout.py \
+    --checkpoint "$CHECKPOINT" \
+    --dataset "$DATASET" \
+    --out "$OUT" \
+    --device cuda \
+    --max-groups "$MAX_GROUPS" \
+    --group-batch-size "$GROUP_BATCH_SIZE" \
+    --sim-backend physx_cuda:0 \
+    --render-backend cpu \
+    --selection-mode field \
+    --num-candidates "$NUM_CANDIDATES" \
+    --candidate-sigma "$CANDIDATE_SIGMA" \
+    --selection-seed "$SELECTION_SEED"
+
+echo ""
+echo "Field-guided rollout complete"
+echo "Results: $OUT"

scripts/slurm/eval_maniskill_policy_rollout.sbatch

@@ -18,8 +18,9 @@ DATASET="${DATASET:?Set DATASET to a ManiSkill CIL dataset or collection}"
 SEED="${SLURM_ARRAY_TASK_ID:-0}"
 RUN_ROOT="${RUN_ROOT:-}"
 OBJECTIVE="${OBJECTIVE:-lattice_field}"
+CHECKPOINT_NAME="${CHECKPOINT_NAME:-best.pt}"
 if [[ -n "$RUN_ROOT" ]]; then
-  CHECKPOINT="${CHECKPOINT:-$RUN_ROOT/$OBJECTIVE/seed_$SEED/best.pt}"
+  CHECKPOINT="${CHECKPOINT:-$RUN_ROOT/$OBJECTIVE/seed_$SEED/$CHECKPOINT_NAME}"
   OUT="${OUT:-$RUN_ROOT/$OBJECTIVE/seed_$SEED/policy_rollout.json}"
 else
   CHECKPOINT="${CHECKPOINT:?Set CHECKPOINT, or RUN_ROOT for seed-indexed array runs}"
@@ -38,6 +39,10 @@ SIM_BACKEND="${SIM_BACKEND:-physx_cuda:0}"
 RENDER_BACKEND="${RENDER_BACKEND:-none}"
 ALL_GROUPS="${ALL_GROUPS:-0}"
 DEVICE="${DEVICE:-cuda}"
+SELECTION_MODE="${SELECTION_MODE:-policy}"
+NUM_CANDIDATES="${NUM_CANDIDATES:-1}"
+CANDIDATE_SIGMA="${CANDIDATE_SIGMA:-0.2}"
+SELECTION_SEED="${SELECTION_SEED:-0}"
 
 module load StdEnv/2023 apptainer/1.4.5
 cd "$PROJECT_DIR"
@@ -73,4 +78,8 @@ apptainer exec --nv \
     --group-batch-size "$GROUP_BATCH_SIZE" \
     --sim-backend "$SIM_BACKEND" \
     --render-backend "$RENDER_BACKEND" \
+    --selection-mode "$SELECTION_MODE" \
+    --num-candidates "$NUM_CANDIDATES" \
+    --candidate-sigma "$CANDIDATE_SIGMA" \
+    --selection-seed "$SELECTION_SEED" \
     "${EXTRA_ARGS[@]}"

scripts/slurm/summarize_h16_field_sweep.sbatch

@@ -0,0 +1,137 @@
+#!/bin/bash
+#SBATCH --job-name=sum_h16_field
+#SBATCH --account=def-yalda
+#SBATCH --time=00:20:00
+#SBATCH --cpus-per-task=1
+#SBATCH --mem=2G
+#SBATCH --output=outputs/hpc/logs/%x_%j.out
+#SBATCH --error=outputs/hpc/logs/%x_%j.err
+
+set -euo pipefail
+
+PROJECT_DIR="${PROJECT_DIR:-$SLURM_SUBMIT_DIR}"
+RUN_ROOT="${RUN_ROOT:-/scratch/$USER/dovla/experiments/dovla_h16_field_sweep}"
+if [[ -z "${PYTHON:-}" ]]; then
+  if [[ -x "$PROJECT_DIR/.venv/bin/python" ]]; then
+    PYTHON="$PROJECT_DIR/.venv/bin/python"
+  else
+    PYTHON="python3"
+  fi
+fi
+
+cd "$PROJECT_DIR"
+mkdir -p results outputs/hpc/logs
+
+"$PYTHON" - <<'PY'
+from __future__ import annotations
+
+import json
+import statistics
+from pathlib import Path
+
+run_root = Path(__import__("os").environ["RUN_ROOT"])
+baseline = 0.2967
+policy_h16 = 0.29739130434782607
+rows = []
+
+for result_path in sorted(run_root.glob("k*_sigma*/seed_*/online_rollout.json")):
+    data = json.loads(result_path.read_text())
+    cfg = result_path.parents[1].name
+    seed_name = result_path.parent.name
+    rows.append(
+        {
+            "config": cfg,
+            "seed": int(seed_name.split("_")[-1]),
+            "path": str(result_path),
+            "selection_mode": data.get("selection_mode"),
+            "num_candidates": data.get("num_candidates"),
+            "candidate_sigma": data.get("candidate_sigma"),
+            "num_groups": data.get("num_groups"),
+            "policy_rollout_success_rate": data.get("policy_rollout_success_rate", 0.0),
+            "policy_rollout_progress": data.get("policy_rollout_progress", 0.0),
+            "oracle_success_rate": data.get("oracle_success_rate", 0.0),
+            "action_mse_to_best": data.get("action_mse_to_best", 0.0),
+            "per_task": data.get("per_task", {}),
+        }
+    )
+
+by_config: dict[str, list[dict]] = {}
+for row in rows:
+    by_config.setdefault(row["config"], []).append(row)
+
+summary_rows = []
+for config, config_rows in sorted(by_config.items()):
+    successes = [row["policy_rollout_success_rate"] for row in config_rows]
+    progresses = [row["policy_rollout_progress"] for row in config_rows]
+    action_mses = [row["action_mse_to_best"] for row in config_rows]
+    summary_rows.append(
+        {
+            "config": config,
+            "completed_seeds": sorted(row["seed"] for row in config_rows),
+            "num_completed": len(config_rows),
+            "mean_success": statistics.mean(successes),
+            "std_success": statistics.stdev(successes) if len(successes) > 1 else 0.0,
+            "mean_progress": statistics.mean(progresses),
+            "mean_action_mse_to_best": statistics.mean(action_mses),
+            "absolute_gain_vs_h4_baseline": statistics.mean(successes) - baseline,
+            "absolute_gain_vs_h16_policy": statistics.mean(successes) - policy_h16,
+        }
+    )
+
+summary_rows.sort(key=lambda row: (row["mean_success"], row["num_completed"]), reverse=True)
+best = summary_rows[0] if summary_rows else None
+
+payload = {
+    "run_root": str(run_root),
+    "baseline_h4_policy_success": baseline,
+    "baseline_h16_policy_success": policy_h16,
+    "num_result_files": len(rows),
+    "configs": summary_rows,
+    "best": best,
+    "rows": rows,
+}
+
+json_path = Path("results/h16_field_sweep_summary.json")
+json_path.write_text(json.dumps(payload, indent=2))
+
+lines = [
+    "# h=16 Field-Guided Rollout Sweep",
+    "",
+    f"Result root: `{run_root}`",
+    f"Completed result files: {len(rows)}",
+    f"Baseline h=4 policy success: {baseline:.2%}",
+    f"Baseline h=16 policy success: {policy_h16:.2%}",
+    "",
+    "| config | seeds | mean success | gain vs h=16 | progress | action MSE |",
+    "|---|---:|---:|---:|---:|---:|",
+]
+for row in summary_rows:
+    lines.append(
+        "| {config} | {num_completed} | {mean_success:.2%} | {gain:+.2%} | "
+        "{mean_progress:.2%} | {mse:.3f} |".format(
+            config=row["config"],
+            num_completed=row["num_completed"],
+            mean_success=row["mean_success"],
+            gain=row["absolute_gain_vs_h16_policy"],
+            mean_progress=row["mean_progress"],
+            mse=row["mean_action_mse_to_best"],
+        )
+    )
+if best is not None:
+    lines.extend(
+        [
+            "",
+            "Best config:",
+            f"- {best['config']}",
+            f"- mean success: {best['mean_success']:.2%}",
+            f"- gain vs h=16 policy: {best['absolute_gain_vs_h16_policy']:+.2%}",
+        ]
+    )
+
+md_path = Path("results/h16_field_sweep_summary.md")
+md_path.write_text("\n".join(lines) + "\n")
+
+print(json.dumps({"best": best, "num_result_files": len(rows)}, indent=2))
+print(f"Wrote {json_path}")
+print(f"Wrote {md_path}")
+PY

tests/test_maniskill_policy_rollout.py

@@ -194,3 +194,27 @@ def test_field_mode_can_prefer_perturbed_candidate() -> None:
     # i.e. strictly away from the bare mean.
     assert float(((actions - mean) ** 2).sum()) > 0.0
     assert index.tolist()[0] != 0
+
+
+def test_field_mode_scores_clamped_candidates() -> None:
+    import torch
+
+    mean = torch.zeros(1, 1, 3)
+    offset = torch.full_like(mean, 10.0)
+    model = _StubModel(torch, mean, best_offset=offset)
+    actions, index = _select_action_chunk(
+        model,
+        observations=torch.zeros(1, 3),
+        instructions=["a"],
+        torch=torch,
+        selection_mode="field",
+        num_candidates=64,
+        candidate_sigma=10.0,
+        selection_seed=7,
+        action_low=torch.full_like(mean, -0.5),
+        action_high=torch.full_like(mean, 0.5),
+    )
+
+    assert float(actions.max()) <= 0.5
+    assert float(actions.min()) >= -0.5
+    assert index.tolist()[0] != 0

tests/test_trainer.py

@@ -45,9 +45,12 @@ def test_trainer_runs_one_epoch_and_writes_checkpoints(tmp_path: Path) -> None:
 
     assert (run_dir / "latest.pt").exists()
     assert (run_dir / "best.pt").exists()
+    assert (run_dir / "best_policy.pt").exists()
     assert "rank_acc" in result["history"][0]["val"]
+    assert "bc_loss" in result["best_policy"]
     metrics = read_json(run_dir / "metrics.json")
     assert "rank_acc" in metrics["history"][0]["val"]
+    assert "best_policy" in metrics
 
 
 def test_field_utility_includes_terminal_success_bonus() -> None: