scripts/slurm/phase_a2_train_large_model.sbatch

#!/bin/bash
#SBATCH --job-name=dovla_large_train
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --mem=64000M
#SBATCH --time=72:00:00
#SBATCH --output=logs/phase_a2_large_train_%j.out
#SBATCH --error=logs/phase_a2_large_train_%j.err
#SBATCH --array=0-2

set -euo pipefail

# Phase A2: Train large capacity model on 10K dataset
# Target: 40%+ policy success (vs current 29.67%)

PROJECT_DIR="${PROJECT_DIR:-$PWD}"
cd "$PROJECT_DIR"

source .venv/bin/activate

DATASET="/scratch/$USER/dovla/experiments/maniskill_presuccess_six_task_collection"
OUT_DIR="/scratch/$USER/dovla/experiments/phase_a2_large_model"
SEED=$SLURM_ARRAY_TASK_ID

mkdir -p "$OUT_DIR/seed_$SEED" logs

echo "=== Phase A2: Training Large Capacity Model ==="
echo "Seed: $SEED"
echo "Hidden dim: 512 (vs current 256)"
echo "Dataset: 10K groups"
echo "Target: 40%+ policy success"
echo ""

python scripts/train_dovla.py \
    --dataset "$DATASET" \
    --out "$OUT_DIR/seed_$SEED" \
    --objective lattice_field \
    --hidden-dim 512 \
    --action-horizon 4 \
    --epochs 100 \
    --batch-groups 16 \
    --records-per-group 8 \
    --lr 0.0003 \
    --weight-decay 0.01 \
    --device auto \
    --seed $SEED \
    --observation-mode state \
    --loss-weight bc=1.0 \
    --loss-weight field_effect=1.0 \
    --loss-weight field_potential=1.0 \
    --loss-weight field_preference=0.5 \
    --loss-weight field_anchor=0.1

echo ""
echo "✅ Phase A2 complete: Large model trained (seed $SEED)"
echo ""
echo "Next: Run phase_a3_eval_large_model.sbatch"