Move DeMemWM multiview launchers to debug scripts

.debug_scripts/train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_2h200_debug.sh

@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=dmwm-mview-debug
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=2
+#SBATCH --gres=gpu:2
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=256G
+#SBATCH --time=02:00:00
+#SBATCH --partition=gpu
+#SBATCH --output=slurm_logs/dmwm-mview-debug-%j.out
+#SBATCH --error=slurm_logs/dmwm-mview-debug-%j.out
+#SBATCH --chdir=/share_1/users/bonan_ding/WorldMem
+
+set -eo pipefail
+
+REPO=/share_1/users/bonan_ding/WorldMem
+PYTHON_BIN=/share_1/users/bonan_ding/.conda/envs/worldmem/bin/python
+LOCAL_OASIS=/share_1/users/bonan_ding/worldmem_ckpt/open-oasis-ckpts/oasis500m.safetensors
+LOCAL_VAE=/share_1/users/bonan_ding/worldmem_ckpt/open-oasis-ckpts/vit-l-20.safetensors
+LOCAL_DATA=/share_1/users/bonan_ding/worldmem_data/minecraft
+LOCAL_FEATURES=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
+
+RUN_NAME=train_dememwm_curric_multiview_anchorclean_causal_cleanmem_debug_${SLURM_JOB_ID:-manual}
+OUTPUT_DIR=${REPO}/outputs/${RUN_NAME}
+LOG_PREFIX="[DEMEMWM_MULTIVIEW_DEBUG]"
+CURRICULUM='{enabled:true,stages:[{name:no_updown,until_step:4,dataset:{wo_updown:true}},{name:with_updown,until_step:8,dataset:{wo_updown:false}}]}'
+
+export PYTHONPATH="${REPO}:${PYTHONPATH:-}"
+export HF_HOME=/share_1/users/bonan_ding/.tmp/cache/huggingface
+export TORCH_HOME=/share_1/users/bonan_ding/.tmp/cache/torch
+export PIP_CACHE_DIR=/share_1/users/bonan_ding/.tmp/cache/pip
+export XDG_CACHE_HOME=/share_1/users/bonan_ding/.tmp/cache/xdg
+export TRITON_CACHE_DIR=/share_1/users/bonan_ding/.tmp/cache/triton
+export TMPDIR=/share_1/users/bonan_ding/.tmp
+export WANDB_MODE=offline
+export WANDB_DISABLED=false
+export TOKENIZERS_PARALLELISM=false
+export HYDRA_FULL_ERROR=1
+export NCCL_P2P_DISABLE=1
+export CUDA_VISIBLE_DEVICES=0,1
+export PYTHONUNBUFFERED=1
+
+mkdir -p "${OUTPUT_DIR}" slurm_logs \
+    "${HF_HOME}" "${TORCH_HOME}" "${PIP_CACHE_DIR}" "${XDG_CACHE_HOME}" "${TRITON_CACHE_DIR}"
+
+cd "${REPO}"
+if [[ ! -s "${LOCAL_OASIS}" ]]; then
+    echo "${LOG_PREFIX}[ERROR] missing diffusion checkpoint: ${LOCAL_OASIS}" >&2
+    exit 2
+fi
+if [[ ! -s "${LOCAL_VAE}" ]]; then
+    echo "${LOG_PREFIX}[ERROR] missing VAE checkpoint: ${LOCAL_VAE}" >&2
+    exit 2
+fi
+if ! find "${LOCAL_DATA}/training" -type f -name '*.mp4' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing raw training mp4s under ${LOCAL_DATA}/training" >&2
+    exit 2
+fi
+if ! find "${LOCAL_DATA}/validation" -type f -name '*.mp4' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing raw validation mp4s under ${LOCAL_DATA}/validation" >&2
+    exit 2
+fi
+if ! find "${LOCAL_FEATURES}/training" -type f -name '*_vae_feature.npy' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing training VAE feature cache under ${LOCAL_FEATURES}/training" >&2
+    exit 2
+fi
+if ! find "${LOCAL_FEATURES}/validation" -type f -name '*_vae_feature.npy' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing validation VAE feature cache under ${LOCAL_FEATURES}/validation" >&2
+    exit 2
+fi
+
+echo "${LOG_PREFIX} repo=${REPO} output=${OUTPUT_DIR} partition=${SLURM_JOB_PARTITION:-local} job=${SLURM_JOB_ID:-manual}"
+echo "${LOG_PREFIX} branch=$(git branch --show-current || true) commit=$(git rev-parse --short HEAD || true)"
+echo "${LOG_PREFIX} status=$(git status --short | tr '\n' ';' || true)"
+echo "${LOG_PREFIX} run_name=${RUN_NAME} wandb_mode=${WANDB_MODE} batch_size_per_gpu=8 global_batch=16"
+echo "${LOG_PREFIX} curriculum=no_updown_until_4_then_with_updown_until_8"
+echo "${LOG_PREFIX} memory_selection=causal_true dynamic_policy=multiview selector=fov_greedy clean_memory"
+echo "${LOG_PREFIX} validation=n_frames_valid_700 context_frames_600 generated_frames_100 limit_batch_1"
+echo "${LOG_PREFIX} data=${LOCAL_DATA} features=${LOCAL_FEATURES}"
+nvidia-smi || true
+wandb offline >/dev/null 2>&1 || true
+
+srun "${PYTHON_BIN}" -m main \
+    +name="${RUN_NAME}" \
+    +output_dir="${OUTPUT_DIR}/" \
+    auto_resume=false \
+    wandb.mode=offline \
+    "experiment.tasks=[training]" \
+    experiment.num_nodes=1 \
+    algorithm=dememwm_base \
+    +customized_load=true \
+    +seperate_load=true \
+    +diffusion_model_path="${LOCAL_OASIS}" \
+    +vae_path="${LOCAL_VAE}" \
+    dataset=video_minecraft_dememwm_latent \
+    dataset.save_dir="${LOCAL_DATA}" \
+    dataset.precomputed_feature_dir="${LOCAL_FEATURES}" \
+    dataset.n_frames=8 \
+    dataset.n_frames_valid=700 \
+    dataset.context_length=100 \
+    dataset.single_eval_clip=true \
+    dataset.memory_selection.enabled=true \
+    dataset.memory_selection.causal=true \
+    dataset.memory_selection.max_anchor_frames=2 \
+    dataset.memory_selection.max_dynamic_frames=4 \
+    dataset.memory_selection.max_revisit_frames=2 \
+    dataset.memory_selection.pose_similarity_threshold=0.6 \
+    dataset.memory_selection.training_use_plucker=true \
+    dataset.memory_selection.training_plucker_weight=1.0 \
+    dataset.memory_selection.fov_overlap_threshold=0.6 \
+    dataset.memory_selection.min_total_selected_coverage=0.1 \
+    dataset.memory_selection.local_context_exclusion_frames=8 \
+    dataset.memory_selection.anchor_diverse_selection=true \
+    +dataset.memory_selection.pose_preselect_topk=32 \
+    +dataset.memory_selection.candidate_chunk_size=0 \
+    dataset.memory_selection.dynamic.selection_policy=multiview \
+    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
+    algorithm.causal=true \
+    algorithm.n_tokens=8 \
+    algorithm.context_frames=600 \
+    algorithm.metric_report_segment=100 \
+    algorithm.chunk_size=1 \
+    algorithm.warmup_steps=4 \
+    algorithm.trainability.train_full_dit=true \
+    algorithm.trainability.full_dit_start_step=4 \
+    algorithm.trainability.lr.memory_modules=4.0e-5 \
+    algorithm.trainability.lr.base_dit=1.0e-5 \
+    algorithm.log_video=true \
+    algorithm.save_local=true \
+    algorithm.diffusion.sampling_timesteps=20 \
+    algorithm.noise_route.anchor=all \
+    algorithm.noise_route.dynamic=all \
+    algorithm.noise_route.revisit=all \
+    algorithm.memory_noise.enabled=false \
+    algorithm.memory_noise.anchor_max_fraction=0.0 \
+    algorithm.memory_noise.dynamic_max_fraction=0.0 \
+    algorithm.memory_noise.revisit_max_fraction=0.0 \
+    +algorithm.memory_noise.validation_noisy_memory=false \
+    "experiment.training.curriculum=${CURRICULUM}" \
+    experiment.training.batch_size=8 \
+    experiment.training.optim.accumulate_grad_batches=1 \
+    experiment.validation.batch_size=1 \
+    experiment.validation.limit_batch=1 \
+    experiment.validation.data.num_workers=2 \
+    experiment.validation.inference_mode=false \
+    experiment.training.checkpointing.every_n_train_steps=4 \
+    experiment.validation.val_every_n_step=4 \
+    experiment.training.max_steps=8

.debug_scripts/train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_full.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+#SBATCH --job-name=dmwm-mview-clean-2h200
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=2
+#SBATCH --gres=gpu:2
+#SBATCH --cpus-per-task=8
+#SBATCH --mem=256G
+#SBATCH --time=72:00:00
+#SBATCH --partition=gpu
+#SBATCH --output=slurm_logs/dmwm-mview-clean-2h200-%j.out
+#SBATCH --error=slurm_logs/dmwm-mview-clean-2h200-%j.out
+#SBATCH --chdir=/share_1/users/bonan_ding/WorldMem
+
+set -eo pipefail
+
+REPO=/share_1/users/bonan_ding/WorldMem
+PYTHON_BIN=/share_1/users/bonan_ding/.conda/envs/worldmem/bin/python
+LOCAL_OASIS=/share_1/users/bonan_ding/worldmem_ckpt/open-oasis-ckpts/oasis500m.safetensors
+LOCAL_VAE=/share_1/users/bonan_ding/worldmem_ckpt/open-oasis-ckpts/vit-l-20.safetensors
+LOCAL_DATA=/share_1/users/bonan_ding/worldmem_data/minecraft
+LOCAL_FEATURES=/share_1/users/bonan_ding/worldmem_data/minecraft/vae_features
+
+RUN_NAME=train_dememwm_curric_multiview_anchorclean_causal_cleanmem_fovgreedy_a2d4r2_route_ntok8_2h200_bs8_global16_400k
+OUTPUT_DIR=${REPO}/outputs/${RUN_NAME}
+LOG_PREFIX="[DEMEMWM_MULTIVIEW_CLEAN_2H200]"
+CURRICULUM='{enabled:true,stages:[{name:no_updown,until_step:120000,dataset:{wo_updown:true}},{name:with_updown,until_step:400000,dataset:{wo_updown:false}}]}'
+
+export PYTHONPATH="${REPO}:${PYTHONPATH:-}"
+export HF_HOME=/share_1/users/bonan_ding/.tmp/cache/huggingface
+export TORCH_HOME=/share_1/users/bonan_ding/.tmp/cache/torch
+export PIP_CACHE_DIR=/share_1/users/bonan_ding/.tmp/cache/pip
+export XDG_CACHE_HOME=/share_1/users/bonan_ding/.tmp/cache/xdg
+export TRITON_CACHE_DIR=/share_1/users/bonan_ding/.tmp/cache/triton
+export TMPDIR=/share_1/users/bonan_ding/.tmp
+export WANDB_MODE=online
+export WANDB_DISABLED=false
+export TOKENIZERS_PARALLELISM=false
+export HYDRA_FULL_ERROR=1
+export NCCL_P2P_DISABLE=1
+export CUDA_VISIBLE_DEVICES=0,1
+export PYTHONUNBUFFERED=1
+
+mkdir -p "${OUTPUT_DIR}" slurm_logs \
+    "${HF_HOME}" "${TORCH_HOME}" "${PIP_CACHE_DIR}" "${XDG_CACHE_HOME}" "${TRITON_CACHE_DIR}"
+
+cd "${REPO}"
+if [[ ! -s "${LOCAL_OASIS}" ]]; then
+    echo "${LOG_PREFIX}[ERROR] missing diffusion checkpoint: ${LOCAL_OASIS}" >&2
+    exit 2
+fi
+if [[ ! -s "${LOCAL_VAE}" ]]; then
+    echo "${LOG_PREFIX}[ERROR] missing VAE checkpoint: ${LOCAL_VAE}" >&2
+    exit 2
+fi
+if ! find "${LOCAL_DATA}/training" -type f -name '*.mp4' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing raw training mp4s under ${LOCAL_DATA}/training" >&2
+    exit 2
+fi
+if ! find "${LOCAL_DATA}/validation" -type f -name '*.mp4' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing raw validation mp4s under ${LOCAL_DATA}/validation" >&2
+    exit 2
+fi
+if ! find "${LOCAL_FEATURES}/training" -type f -name '*_vae_feature.npy' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing training VAE feature cache under ${LOCAL_FEATURES}/training" >&2
+    exit 2
+fi
+if ! find "${LOCAL_FEATURES}/validation" -type f -name '*_vae_feature.npy' -print -quit 2>/dev/null | grep -q .; then
+    echo "${LOG_PREFIX}[ERROR] missing validation VAE feature cache under ${LOCAL_FEATURES}/validation" >&2
+    exit 2
+fi
+
+echo "${LOG_PREFIX} repo=${REPO} output=${OUTPUT_DIR} partition=${SLURM_JOB_PARTITION:-local} job=${SLURM_JOB_ID:-manual}"
+echo "${LOG_PREFIX} branch=$(git branch --show-current || true) commit=$(git rev-parse --short HEAD || true)"
+echo "${LOG_PREFIX} status=$(git status --short | tr '\n' ';' || true)"
+echo "${LOG_PREFIX} run_name=${RUN_NAME} wandb_mode=${WANDB_MODE} batch_size_per_gpu=8 global_batch=16"
+echo "${LOG_PREFIX} curriculum=no_updown_until_120000_then_with_updown_until_400000"
+echo "${LOG_PREFIX} memory_selection=causal_true dynamic_policy=multiview selector=fov_greedy clean_memory"
+echo "${LOG_PREFIX} data=${LOCAL_DATA} features=${LOCAL_FEATURES}"
+nvidia-smi || true
+
+srun "${PYTHON_BIN}" -m main \
+    +name="${RUN_NAME}" \
+    +output_dir="${OUTPUT_DIR}/" \
+    auto_resume=true \
+    wandb.mode=online \
+    "experiment.tasks=[training]" \
+    experiment.num_nodes=1 \
+    algorithm=dememwm_base \
+    +customized_load=true \
+    +seperate_load=true \
+    +diffusion_model_path="${LOCAL_OASIS}" \
+    +vae_path="${LOCAL_VAE}" \
+    dataset=video_minecraft_dememwm_latent \
+    dataset.save_dir="${LOCAL_DATA}" \
+    dataset.precomputed_feature_dir="${LOCAL_FEATURES}" \
+    dataset.n_frames=8 \
+    dataset.n_frames_valid=1100 \
+    dataset.context_length=100 \
+    dataset.single_eval_clip=true \
+    dataset.memory_selection.enabled=true \
+    dataset.memory_selection.causal=true \
+    dataset.memory_selection.max_anchor_frames=2 \
+    dataset.memory_selection.max_dynamic_frames=4 \
+    dataset.memory_selection.max_revisit_frames=2 \
+    dataset.memory_selection.pose_similarity_threshold=0.6 \
+    dataset.memory_selection.training_use_plucker=true \
+    dataset.memory_selection.training_plucker_weight=1.0 \
+    dataset.memory_selection.fov_overlap_threshold=0.6 \
+    dataset.memory_selection.min_total_selected_coverage=0.1 \
+    dataset.memory_selection.local_context_exclusion_frames=8 \
+    dataset.memory_selection.anchor_diverse_selection=true \
+    +dataset.memory_selection.pose_preselect_topk=64 \
+    +dataset.memory_selection.candidate_chunk_size=64 \
+    dataset.memory_selection.dynamic.selection_policy=multiview \
+    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
+    algorithm.causal=true \
+    algorithm.n_tokens=8 \
+    algorithm.context_frames=600 \
+    algorithm.metric_report_segment=100 \
+    algorithm.chunk_size=1 \
+    algorithm.warmup_steps=10000 \
+    algorithm.trainability.train_full_dit=true \
+    algorithm.trainability.full_dit_start_step=40000 \
+    algorithm.trainability.lr.memory_modules=4.0e-5 \
+    algorithm.trainability.lr.base_dit=1.0e-5 \
+    algorithm.log_video=true \
+    algorithm.save_local=true \
+    algorithm.diffusion.sampling_timesteps=20 \
+    algorithm.noise_route.anchor=all \
+    algorithm.noise_route.dynamic=all \
+    algorithm.noise_route.revisit=all \
+    algorithm.memory_noise.enabled=false \
+    algorithm.memory_noise.anchor_max_fraction=0.0 \
+    algorithm.memory_noise.dynamic_max_fraction=0.0 \
+    algorithm.memory_noise.revisit_max_fraction=0.0 \
+    +algorithm.memory_noise.validation_noisy_memory=false \
+    "experiment.training.curriculum=${CURRICULUM}" \
+    experiment.training.batch_size=8 \
+    experiment.training.optim.accumulate_grad_batches=1 \
+    experiment.validation.batch_size=1 \
+    experiment.validation.limit_batch=4 \
+    experiment.training.checkpointing.every_n_train_steps=5000 \
+    experiment.validation.val_every_n_step=5000 \
+    experiment.training.max_steps=400000

train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_2h200_debug.sh

@@ -1,101 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export PYTHONPATH="./:${PYTHONPATH:-}"
-export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1}
-
-NPROC_PER_NODE=${NPROC_PER_NODE:-2}
-DATA_ROOT=${DATA_ROOT:-data/minecraft_simple_backforward}
-FEATURE_ROOT=${FEATURE_ROOT:-${DATA_ROOT}/vae_features}
-OUTPUT_ROOT=${OUTPUT_ROOT:-outputs}
-RUN_NAME=${RUN_NAME:-train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_2h200_debug}
-OUTPUT_DIR=${OUTPUT_DIR:-${OUTPUT_ROOT}/${RUN_NAME}}
-
-: "${DIFFUSION_MODEL_PATH:?Set DIFFUSION_MODEL_PATH to a local diffusion checkpoint}"
-: "${VAE_PATH:?Set VAE_PATH to a local VAE checkpoint}"
-
-MAX_STEPS=${MAX_STEPS:-50}
-VAL_EVERY_N_STEP=${VAL_EVERY_N_STEP:-25}
-CHECKPOINT_EVERY_N_STEPS=${CHECKPOINT_EVERY_N_STEPS:-25}
-LIMIT_BATCH=${LIMIT_BATCH:-1}
-N_FRAMES_VALID=${N_FRAMES_VALID:-128}
-CONTEXT_LENGTH=${CONTEXT_LENGTH:-100}
-ALGORITHM_CONTEXT_FRAMES=${ALGORITHM_CONTEXT_FRAMES:-100}
-TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-1}
-VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-1}
-NUM_WORKERS=${NUM_WORKERS:-2}
-CURRICULUM_NO_UPDOWN_UNTIL_STEP=${CURRICULUM_NO_UPDOWN_UNTIL_STEP:-25}
-CURRICULUM="{enabled:true,stages:[{name:no_updown,until_step:${CURRICULUM_NO_UPDOWN_UNTIL_STEP},dataset:{wo_updown:true}},{name:with_updown,until_step:${MAX_STEPS},dataset:{wo_updown:false}}]}"
-
-mkdir -p "${OUTPUT_DIR}"
-
-torchrun --standalone --nproc_per_node=${NPROC_PER_NODE} -m main \
-    "+name=${RUN_NAME}" \
-    "+output_dir=${OUTPUT_DIR}/" \
-    auto_resume=false \
-    wandb.mode=offline \
-    "experiment.tasks=[training]" \
-    algorithm=dememwm_base \
-    +customized_load=true \
-    +seperate_load=true \
-    "+diffusion_model_path=${DIFFUSION_MODEL_PATH}" \
-    "+vae_path=${VAE_PATH}" \
-    dataset=video_minecraft_dememwm_latent \
-    "dataset.save_dir=${DATA_ROOT}" \
-    "dataset.precomputed_feature_dir=${FEATURE_ROOT}" \
-    dataset.n_frames=8 \
-    "dataset.n_frames_valid=${N_FRAMES_VALID}" \
-    "dataset.context_length=${CONTEXT_LENGTH}" \
-    dataset.single_eval_clip=true \
-    dataset.memory_selection.enabled=true \
-    dataset.memory_selection.max_anchor_frames=2 \
-    dataset.memory_selection.max_dynamic_frames=4 \
-    dataset.memory_selection.max_revisit_frames=2 \
-    dataset.memory_selection.causal=true \
-    dataset.memory_selection.dynamic.selection_policy=multiview \
-    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
-    dataset.memory_selection.pose_similarity_threshold=0.6 \
-    dataset.memory_selection.training_use_plucker=true \
-    dataset.memory_selection.training_plucker_weight=1.0 \
-    dataset.memory_selection.fov_overlap_threshold=0.6 \
-    dataset.memory_selection.min_total_selected_coverage=0.1 \
-    dataset.memory_selection.local_context_exclusion_frames=8 \
-    dataset.memory_selection.anchor_diverse_selection=true \
-    dataset.memory_selection.pose_preselect_topk=16 \
-    dataset.memory_selection.candidate_chunk_size=16 \
-    algorithm.causal=true \
-    algorithm.n_tokens=8 \
-    "algorithm.context_frames=${ALGORITHM_CONTEXT_FRAMES}" \
-    algorithm.metric_report_segment=0 \
-    +algorithm.log_per_frame_metrics=false \
-    algorithm.chunk_size=1 \
-    algorithm.warmup_steps=100 \
-    algorithm.trainability.train_full_dit=true \
-    algorithm.trainability.full_dit_start_step=25 \
-    algorithm.trainability.lr.memory_modules=4.0e-5 \
-    algorithm.trainability.lr.base_dit=1.0e-5 \
-    algorithm.log_video=false \
-    algorithm.save_local=true \
-    algorithm.diffusion.sampling_timesteps=20 \
-    algorithm.noise_route.anchor=all \
-    algorithm.noise_route.dynamic=all \
-    algorithm.noise_route.revisit=all \
-    algorithm.memory_noise.enabled=false \
-    algorithm.memory_noise.anchor_max_fraction=0.0 \
-    algorithm.memory_noise.dynamic_max_fraction=0.0 \
-    algorithm.memory_noise.revisit_max_fraction=0.0 \
-    +algorithm.memory_noise.validation_noisy_memory=false \
-    "experiment.training.curriculum=${CURRICULUM}" \
-    "experiment.training.batch_size=${TRAIN_BATCH_SIZE}" \
-    experiment.training.precision=16-mixed \
-    experiment.training.optim.accumulate_grad_batches=1 \
-    experiment.training.optim.gradient_clip_val=1.0 \
-    "experiment.training.data.num_workers=${NUM_WORKERS}" \
-    "experiment.validation.data.num_workers=${NUM_WORKERS}" \
-    "experiment.validation.batch_size=${VAL_BATCH_SIZE}" \
-    "experiment.validation.limit_batch=${LIMIT_BATCH}" \
-    "experiment.validation.val_every_n_step=${VAL_EVERY_N_STEP}" \
-    "experiment.training.checkpointing.every_n_train_steps=${CHECKPOINT_EVERY_N_STEPS}" \
-    experiment.training.checkpointing.save_last_k=2 \
-    "experiment.training.max_steps=${MAX_STEPS}"

train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_full.sh

@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-export PYTHONPATH="./:${PYTHONPATH:-}"
-export HYDRA_FULL_ERROR=1
-export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1}
-
-NPROC_PER_NODE=${NPROC_PER_NODE:-2}
-DATA_ROOT=${DATA_ROOT:-data/minecraft_simple_backforward}
-FEATURE_ROOT=${FEATURE_ROOT:-${DATA_ROOT}/vae_features}
-OUTPUT_ROOT=${OUTPUT_ROOT:-outputs}
-RUN_NAME=${RUN_NAME:-train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_full}
-OUTPUT_DIR=${OUTPUT_DIR:-${OUTPUT_ROOT}/${RUN_NAME}}
-
-: "${DIFFUSION_MODEL_PATH:?Set DIFFUSION_MODEL_PATH to a local diffusion checkpoint}"
-: "${VAE_PATH:?Set VAE_PATH to a local VAE checkpoint}"
-
-MAX_STEPS=${MAX_STEPS:-120000}
-VAL_EVERY_N_STEP=${VAL_EVERY_N_STEP:-2000}
-CHECKPOINT_EVERY_N_STEPS=${CHECKPOINT_EVERY_N_STEPS:-2000}
-LIMIT_BATCH=${LIMIT_BATCH:-1}
-N_FRAMES_VALID=${N_FRAMES_VALID:-1100}
-CONTEXT_LENGTH=${CONTEXT_LENGTH:-100}
-ALGORITHM_CONTEXT_FRAMES=${ALGORITHM_CONTEXT_FRAMES:-600}
-TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-16}
-VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-1}
-ACCUMULATE_GRAD_BATCHES=${ACCUMULATE_GRAD_BATCHES:-1}
-NUM_WORKERS=${NUM_WORKERS:-4}
-WANDB_MODE=${WANDB_MODE:-offline}
-CURRICULUM_NO_UPDOWN_UNTIL_STEP=${CURRICULUM_NO_UPDOWN_UNTIL_STEP:-60000}
-CURRICULUM="{enabled:true,stages:[{name:no_updown,until_step:${CURRICULUM_NO_UPDOWN_UNTIL_STEP},dataset:{wo_updown:true}},{name:with_updown,until_step:${MAX_STEPS},dataset:{wo_updown:false}}]}"
-
-mkdir -p "${OUTPUT_DIR}"
-
-torchrun --standalone --nproc_per_node=${NPROC_PER_NODE} -m main \
-    "+name=${RUN_NAME}" \
-    "+output_dir=${OUTPUT_DIR}/" \
-    auto_resume=true \
-    "wandb.mode=${WANDB_MODE}" \
-    "experiment.tasks=[training]" \
-    algorithm=dememwm_base \
-    +customized_load=true \
-    +seperate_load=true \
-    "+diffusion_model_path=${DIFFUSION_MODEL_PATH}" \
-    "+vae_path=${VAE_PATH}" \
-    dataset=video_minecraft_dememwm_latent \
-    "dataset.save_dir=${DATA_ROOT}" \
-    "dataset.precomputed_feature_dir=${FEATURE_ROOT}" \
-    dataset.n_frames=8 \
-    "dataset.n_frames_valid=${N_FRAMES_VALID}" \
-    "dataset.context_length=${CONTEXT_LENGTH}" \
-    dataset.single_eval_clip=true \
-    dataset.memory_selection.enabled=true \
-    dataset.memory_selection.max_anchor_frames=2 \
-    dataset.memory_selection.max_dynamic_frames=4 \
-    dataset.memory_selection.max_revisit_frames=2 \
-    dataset.memory_selection.causal=true \
-    dataset.memory_selection.dynamic.selection_policy=multiview \
-    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
-    dataset.memory_selection.pose_similarity_threshold=0.6 \
-    dataset.memory_selection.training_use_plucker=true \
-    dataset.memory_selection.training_plucker_weight=1.0 \
-    dataset.memory_selection.fov_overlap_threshold=0.6 \
-    dataset.memory_selection.min_total_selected_coverage=0.1 \
-    dataset.memory_selection.local_context_exclusion_frames=8 \
-    dataset.memory_selection.anchor_diverse_selection=true \
-    dataset.memory_selection.pose_preselect_topk=64 \
-    dataset.memory_selection.candidate_chunk_size=64 \
-    algorithm.causal=true \
-    algorithm.n_tokens=8 \
-    "algorithm.context_frames=${ALGORITHM_CONTEXT_FRAMES}" \
-    algorithm.metric_report_segment=100 \
-    +algorithm.log_per_frame_metrics=true \
-    algorithm.chunk_size=1 \
-    algorithm.warmup_steps=5000 \
-    algorithm.trainability.train_full_dit=true \
-    algorithm.trainability.full_dit_start_step=40000 \
-    algorithm.trainability.lr.memory_modules=4.0e-5 \
-    algorithm.trainability.lr.base_dit=1.0e-5 \
-    algorithm.log_video=true \
-    algorithm.save_local=true \
-    algorithm.diffusion.sampling_timesteps=20 \
-    algorithm.noise_route.anchor=all \
-    algorithm.noise_route.dynamic=all \
-    algorithm.noise_route.revisit=all \
-    algorithm.memory_noise.enabled=false \
-    algorithm.memory_noise.anchor_max_fraction=0.0 \
-    algorithm.memory_noise.dynamic_max_fraction=0.0 \
-    algorithm.memory_noise.revisit_max_fraction=0.0 \
-    +algorithm.memory_noise.validation_noisy_memory=false \
-    "experiment.training.curriculum=${CURRICULUM}" \
-    "experiment.training.batch_size=${TRAIN_BATCH_SIZE}" \
-    experiment.training.precision=16-mixed \
-    "experiment.training.optim.accumulate_grad_batches=${ACCUMULATE_GRAD_BATCHES}" \
-    experiment.training.optim.gradient_clip_val=1.0 \
-    "experiment.training.data.num_workers=${NUM_WORKERS}" \
-    "experiment.validation.data.num_workers=${NUM_WORKERS}" \
-    "experiment.validation.batch_size=${VAL_BATCH_SIZE}" \
-    "experiment.validation.limit_batch=${LIMIT_BATCH}" \
-    "experiment.validation.val_every_n_step=${VAL_EVERY_N_STEP}" \
-    "experiment.training.checkpointing.every_n_train_steps=${CHECKPOINT_EVERY_N_STEPS}" \
-    +experiment.training.checkpointing.save_top_k=5 \
-    +experiment.training.checkpointing.monitor=lpips \
-    +experiment.training.checkpointing.mode=min \
-    "experiment.training.max_steps=${MAX_STEPS}"