#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GPU_IDS="${GPU_IDS:-0,1,2,3,4,5,6,7}"
PRED_ONLY_OUTPUT="${PRED_ONLY_OUTPUT:-/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_dmd_predictor_only}"
JOINT_OUTPUT="${JOINT_OUTPUT:-/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_dmd_joint}"
ORIGINAL_FULL="$ROOT_DIR/checkpoints/self_forcing_dmd.pt"
ORIGINAL_VALIDATION_FFFF="$ROOT_DIR/videos/moviegenvideobench_extended_101_200_dmd_seed0"
PRED_ONLY_PROJECT="$ROOT_DIR/checkpoints/predictor_v4_dmd_predictor_only"
JOINT_PROJECT="$ROOT_DIR/checkpoints/predictor_v4_dmd_joint"

cd "$ROOT_DIR"
exec 9>"$ROOT_DIR/evaluation_results/predictor_v4_dmd_suite.lock"
if ! flock -n 9; then
  echo "错误：已有 Predictor-v4 DMD suite 正在运行。" >&2
  exit 1
fi

GPU_IDS="$GPU_IDS" \
CONFIG_PATH="$ROOT_DIR/configs/predictor_v4_dmd_predictor_only.yaml" \
OUTPUT_DIR="$PRED_ONLY_OUTPUT" \
MASTER_PORT=29741 \
bash scripts/run_predictor_v4_dmd_training.sh

mkdir -p "$PRED_ONLY_PROJECT"
rsync -a \
  "$PRED_ONLY_OUTPUT/checkpoint_final/" \
  "$PRED_ONLY_PROJECT/checkpoint_final/"
rsync -a \
  "$PRED_ONLY_OUTPUT/train_config.json" \
  "$PRED_ONLY_OUTPUT/train_log.jsonl" \
  "$PRED_ONLY_PROJECT/"

GPU_IDS="$GPU_IDS" \
EXPERIMENT_NAME=predictor_v4_dmd_predictor_only \
FULL_CHECKPOINT="$ORIGINAL_FULL" \
PREDICTOR_CHECKPOINT="$PRED_ONLY_OUTPUT/checkpoint_final/predictor_ema.safetensors" \
INCLUDE_FFFF=0 \
REFERENCE_FFFF="$ORIGINAL_VALIDATION_FFFF" \
bash scripts/run_predictor_v4_dmd_validation.sh

GPU_IDS="$GPU_IDS" \
CONFIG_PATH="$ROOT_DIR/configs/predictor_v4_dmd_joint.yaml" \
OUTPUT_DIR="$JOINT_OUTPUT" \
MASTER_PORT=29742 \
bash scripts/run_predictor_v4_dmd_training.sh

mkdir -p "$JOINT_PROJECT"
rsync -a \
  "$JOINT_OUTPUT/checkpoint_final/" \
  "$JOINT_PROJECT/checkpoint_final/"
rsync -a \
  "$JOINT_OUTPUT/train_config.json" \
  "$JOINT_OUTPUT/train_log.jsonl" \
  "$JOINT_PROJECT/"

GPU_IDS="$GPU_IDS" \
EXPERIMENT_NAME=predictor_v4_dmd_joint \
FULL_CHECKPOINT="$JOINT_OUTPUT/checkpoint_final/model.pt" \
PREDICTOR_CHECKPOINT="$JOINT_OUTPUT/checkpoint_final/predictor_ema.safetensors" \
INCLUDE_FFFF=1 \
bash scripts/run_predictor_v4_dmd_validation.sh

echo "[$(date -u +%FT%TZ)] Predictor-only 与 Joint DMD 训练评估全部完成。"