File size: 2,299 Bytes
2847d0b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | #!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GPU_IDS="${GPU_IDS:-0,1,2,3,4,5,6,7}"
PRED_ONLY_OUTPUT="${PRED_ONLY_OUTPUT:-/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_dmd_predictor_only}"
JOINT_OUTPUT="${JOINT_OUTPUT:-/mnt/local_nvme/zoubin/cz/self_forcing_predictor_v4_dmd_joint}"
ORIGINAL_FULL="$ROOT_DIR/checkpoints/self_forcing_dmd.pt"
ORIGINAL_VALIDATION_FFFF="$ROOT_DIR/videos/moviegenvideobench_extended_101_200_dmd_seed0"
PRED_ONLY_PROJECT="$ROOT_DIR/checkpoints/predictor_v4_dmd_predictor_only"
JOINT_PROJECT="$ROOT_DIR/checkpoints/predictor_v4_dmd_joint"
cd "$ROOT_DIR"
exec 9>"$ROOT_DIR/evaluation_results/predictor_v4_dmd_suite.lock"
if ! flock -n 9; then
echo "错误：已有 Predictor-v4 DMD suite 正在运行。" >&2
exit 1
fi
GPU_IDS="$GPU_IDS" \
CONFIG_PATH="$ROOT_DIR/configs/predictor_v4_dmd_predictor_only.yaml" \
OUTPUT_DIR="$PRED_ONLY_OUTPUT" \
MASTER_PORT=29741 \
bash scripts/run_predictor_v4_dmd_training.sh
mkdir -p "$PRED_ONLY_PROJECT"
rsync -a \
"$PRED_ONLY_OUTPUT/checkpoint_final/" \
"$PRED_ONLY_PROJECT/checkpoint_final/"
rsync -a \
"$PRED_ONLY_OUTPUT/train_config.json" \
"$PRED_ONLY_OUTPUT/train_log.jsonl" \
"$PRED_ONLY_PROJECT/"
GPU_IDS="$GPU_IDS" \
EXPERIMENT_NAME=predictor_v4_dmd_predictor_only \
FULL_CHECKPOINT="$ORIGINAL_FULL" \
PREDICTOR_CHECKPOINT="$PRED_ONLY_OUTPUT/checkpoint_final/predictor_ema.safetensors" \
INCLUDE_FFFF=0 \
REFERENCE_FFFF="$ORIGINAL_VALIDATION_FFFF" \
bash scripts/run_predictor_v4_dmd_validation.sh
GPU_IDS="$GPU_IDS" \
CONFIG_PATH="$ROOT_DIR/configs/predictor_v4_dmd_joint.yaml" \
OUTPUT_DIR="$JOINT_OUTPUT" \
MASTER_PORT=29742 \
bash scripts/run_predictor_v4_dmd_training.sh
mkdir -p "$JOINT_PROJECT"
rsync -a \
"$JOINT_OUTPUT/checkpoint_final/" \
"$JOINT_PROJECT/checkpoint_final/"
rsync -a \
"$JOINT_OUTPUT/train_config.json" \
"$JOINT_OUTPUT/train_log.jsonl" \
"$JOINT_PROJECT/"
GPU_IDS="$GPU_IDS" \
EXPERIMENT_NAME=predictor_v4_dmd_joint \
FULL_CHECKPOINT="$JOINT_OUTPUT/checkpoint_final/model.pt" \
PREDICTOR_CHECKPOINT="$JOINT_OUTPUT/checkpoint_final/predictor_ema.safetensors" \
INCLUDE_FFFF=1 \
bash scripts/run_predictor_v4_dmd_validation.sh
echo "[$(date -u +%FT%TZ)] Predictor-only 与 Joint DMD 训练评估全部完成。"
|