scripts/resume_predictor_v4_training_and_sync.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DATASET_ROOT="/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_1000_seed0"
NVME_TRAIN_ROOT="/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_training_1000_seed0"
RUN_DIR="$ROOT_DIR/checkpoints/predictor_v4_1000_seed0"

cd "$ROOT_DIR"
mkdir -p "$NVME_TRAIN_ROOT" "$RUN_DIR"
exec 9>"$RUN_DIR/training_and_sync.lock"
if ! flock -n 9; then
  echo "错误：已有 Predictor-v4 training/sync 任务正在运行。" >&2
  exit 1
fi
echo "$$" >"$RUN_DIR/training_and_sync.pid"
trap 'rm -f "$RUN_DIR/training_and_sync.pid"' EXIT

if [[ "$(wc -l < "$DATASET_ROOT/manifest.jsonl")" -ne 7000 ]]; then
  echo "错误：manifest.jsonl 不完整。" >&2
  exit 1
fi
if [[ "$(wc -l < "$DATASET_ROOT/train_manifest.jsonl")" -ne 6000 ]]; then
  echo "错误：train_manifest.jsonl 不完整。" >&2
  exit 1
fi

echo "[$(date -u +%FT%TZ)] 从完整 NVMe 数据启动 8 GPU Predictor-v4 训练。"
NPROC_PER_NODE=8 \
LOG_DIR="$NVME_TRAIN_ROOT" \
bash scripts/run_predictor_v4_training.sh

echo "[$(date -u +%FT%TZ)] 训练完成，开始同步 NVMe 数据和 checkpoint。"
DATASET_SOURCE="$DATASET_ROOT" \
TRAIN_SOURCE="$NVME_TRAIN_ROOT" \
bash scripts/sync_predictor_v4_artifacts.sh
echo "[$(date -u +%FT%TZ)] 训练与同步全部完成。"