Add local DeMemWM multiview launchers

.exp_artifact/dememwm_dynamic_multiview_memory_selection_plan.md

@@ -897,7 +897,7 @@ Add DeMemWM multiview selector benchmark
 
 ## Substep 7: Add Local Launchers And Final Validation
 
-Status: `[ ]`
+Status: `[x]`
 
 Goal:
 

train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_2h200_debug.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export PYTHONPATH="./:${PYTHONPATH:-}"
+export HYDRA_FULL_ERROR=1
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1}
+
+NPROC_PER_NODE=${NPROC_PER_NODE:-2}
+DATA_ROOT=${DATA_ROOT:-data/minecraft_simple_backforward}
+FEATURE_ROOT=${FEATURE_ROOT:-${DATA_ROOT}/vae_features}
+OUTPUT_ROOT=${OUTPUT_ROOT:-outputs}
+RUN_NAME=${RUN_NAME:-train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_2h200_debug}
+OUTPUT_DIR=${OUTPUT_DIR:-${OUTPUT_ROOT}/${RUN_NAME}}
+
+: "${DIFFUSION_MODEL_PATH:?Set DIFFUSION_MODEL_PATH to a local diffusion checkpoint}"
+: "${VAE_PATH:?Set VAE_PATH to a local VAE checkpoint}"
+
+MAX_STEPS=${MAX_STEPS:-50}
+VAL_EVERY_N_STEP=${VAL_EVERY_N_STEP:-25}
+CHECKPOINT_EVERY_N_STEPS=${CHECKPOINT_EVERY_N_STEPS:-25}
+LIMIT_BATCH=${LIMIT_BATCH:-1}
+N_FRAMES_VALID=${N_FRAMES_VALID:-128}
+CONTEXT_LENGTH=${CONTEXT_LENGTH:-100}
+ALGORITHM_CONTEXT_FRAMES=${ALGORITHM_CONTEXT_FRAMES:-100}
+TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-1}
+VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-1}
+NUM_WORKERS=${NUM_WORKERS:-2}
+CURRICULUM_NO_UPDOWN_UNTIL_STEP=${CURRICULUM_NO_UPDOWN_UNTIL_STEP:-25}
+CURRICULUM="{enabled:true,stages:[{name:no_updown,until_step:${CURRICULUM_NO_UPDOWN_UNTIL_STEP},dataset:{wo_updown:true}},{name:with_updown,until_step:${MAX_STEPS},dataset:{wo_updown:false}}]}"
+
+mkdir -p "${OUTPUT_DIR}"
+
+torchrun --standalone --nproc_per_node=${NPROC_PER_NODE} -m main \
+    "+name=${RUN_NAME}" \
+    "+output_dir=${OUTPUT_DIR}/" \
+    auto_resume=false \
+    wandb.mode=offline \
+    "experiment.tasks=[training]" \
+    algorithm=dememwm_base \
+    +customized_load=true \
+    +seperate_load=true \
+    "+diffusion_model_path=${DIFFUSION_MODEL_PATH}" \
+    "+vae_path=${VAE_PATH}" \
+    dataset=video_minecraft_dememwm_latent \
+    "dataset.save_dir=${DATA_ROOT}" \
+    "dataset.precomputed_feature_dir=${FEATURE_ROOT}" \
+    dataset.n_frames=8 \
+    "dataset.n_frames_valid=${N_FRAMES_VALID}" \
+    "dataset.context_length=${CONTEXT_LENGTH}" \
+    dataset.single_eval_clip=true \
+    dataset.memory_selection.enabled=true \
+    dataset.memory_selection.max_anchor_frames=2 \
+    dataset.memory_selection.max_dynamic_frames=4 \
+    dataset.memory_selection.max_revisit_frames=2 \
+    dataset.memory_selection.causal=true \
+    dataset.memory_selection.dynamic.selection_policy=multiview \
+    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
+    dataset.memory_selection.pose_similarity_threshold=0.6 \
+    dataset.memory_selection.training_use_plucker=true \
+    dataset.memory_selection.training_plucker_weight=1.0 \
+    dataset.memory_selection.fov_overlap_threshold=0.6 \
+    dataset.memory_selection.min_total_selected_coverage=0.1 \
+    dataset.memory_selection.local_context_exclusion_frames=8 \
+    dataset.memory_selection.anchor_diverse_selection=true \
+    dataset.memory_selection.pose_preselect_topk=16 \
+    dataset.memory_selection.candidate_chunk_size=16 \
+    algorithm.causal=true \
+    algorithm.n_tokens=8 \
+    "algorithm.context_frames=${ALGORITHM_CONTEXT_FRAMES}" \
+    algorithm.metric_report_segment=0 \
+    +algorithm.log_per_frame_metrics=false \
+    algorithm.chunk_size=1 \
+    algorithm.warmup_steps=100 \
+    algorithm.trainability.train_full_dit=true \
+    algorithm.trainability.full_dit_start_step=25 \
+    algorithm.trainability.lr.memory_modules=4.0e-5 \
+    algorithm.trainability.lr.base_dit=1.0e-5 \
+    algorithm.log_video=false \
+    algorithm.save_local=true \
+    algorithm.diffusion.sampling_timesteps=20 \
+    algorithm.noise_route.anchor=all \
+    algorithm.noise_route.dynamic=all \
+    algorithm.noise_route.revisit=all \
+    algorithm.memory_noise.enabled=false \
+    algorithm.memory_noise.anchor_max_fraction=0.0 \
+    algorithm.memory_noise.dynamic_max_fraction=0.0 \
+    algorithm.memory_noise.revisit_max_fraction=0.0 \
+    +algorithm.memory_noise.validation_noisy_memory=false \
+    "experiment.training.curriculum=${CURRICULUM}" \
+    "experiment.training.batch_size=${TRAIN_BATCH_SIZE}" \
+    experiment.training.precision=16-mixed \
+    experiment.training.optim.accumulate_grad_batches=1 \
+    experiment.training.optim.gradient_clip_val=1.0 \
+    "experiment.training.data.num_workers=${NUM_WORKERS}" \
+    "experiment.validation.data.num_workers=${NUM_WORKERS}" \
+    "experiment.validation.batch_size=${VAL_BATCH_SIZE}" \
+    "experiment.validation.limit_batch=${LIMIT_BATCH}" \
+    "experiment.validation.val_every_n_step=${VAL_EVERY_N_STEP}" \
+    "experiment.training.checkpointing.every_n_train_steps=${CHECKPOINT_EVERY_N_STEPS}" \
+    experiment.training.checkpointing.save_last_k=2 \
+    "experiment.training.max_steps=${MAX_STEPS}"

train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_full.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export PYTHONPATH="./:${PYTHONPATH:-}"
+export HYDRA_FULL_ERROR=1
+export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1}
+
+NPROC_PER_NODE=${NPROC_PER_NODE:-2}
+DATA_ROOT=${DATA_ROOT:-data/minecraft_simple_backforward}
+FEATURE_ROOT=${FEATURE_ROOT:-${DATA_ROOT}/vae_features}
+OUTPUT_ROOT=${OUTPUT_ROOT:-outputs}
+RUN_NAME=${RUN_NAME:-train_dememwm_curric_multiview_anchorclean_causal_cleanmem_local_full}
+OUTPUT_DIR=${OUTPUT_DIR:-${OUTPUT_ROOT}/${RUN_NAME}}
+
+: "${DIFFUSION_MODEL_PATH:?Set DIFFUSION_MODEL_PATH to a local diffusion checkpoint}"
+: "${VAE_PATH:?Set VAE_PATH to a local VAE checkpoint}"
+
+MAX_STEPS=${MAX_STEPS:-120000}
+VAL_EVERY_N_STEP=${VAL_EVERY_N_STEP:-2000}
+CHECKPOINT_EVERY_N_STEPS=${CHECKPOINT_EVERY_N_STEPS:-2000}
+LIMIT_BATCH=${LIMIT_BATCH:-1}
+N_FRAMES_VALID=${N_FRAMES_VALID:-1100}
+CONTEXT_LENGTH=${CONTEXT_LENGTH:-100}
+ALGORITHM_CONTEXT_FRAMES=${ALGORITHM_CONTEXT_FRAMES:-600}
+TRAIN_BATCH_SIZE=${TRAIN_BATCH_SIZE:-16}
+VAL_BATCH_SIZE=${VAL_BATCH_SIZE:-1}
+ACCUMULATE_GRAD_BATCHES=${ACCUMULATE_GRAD_BATCHES:-1}
+NUM_WORKERS=${NUM_WORKERS:-4}
+WANDB_MODE=${WANDB_MODE:-offline}
+CURRICULUM_NO_UPDOWN_UNTIL_STEP=${CURRICULUM_NO_UPDOWN_UNTIL_STEP:-60000}
+CURRICULUM="{enabled:true,stages:[{name:no_updown,until_step:${CURRICULUM_NO_UPDOWN_UNTIL_STEP},dataset:{wo_updown:true}},{name:with_updown,until_step:${MAX_STEPS},dataset:{wo_updown:false}}]}"
+
+mkdir -p "${OUTPUT_DIR}"
+
+torchrun --standalone --nproc_per_node=${NPROC_PER_NODE} -m main \
+    "+name=${RUN_NAME}" \
+    "+output_dir=${OUTPUT_DIR}/" \
+    auto_resume=true \
+    "wandb.mode=${WANDB_MODE}" \
+    "experiment.tasks=[training]" \
+    algorithm=dememwm_base \
+    +customized_load=true \
+    +seperate_load=true \
+    "+diffusion_model_path=${DIFFUSION_MODEL_PATH}" \
+    "+vae_path=${VAE_PATH}" \
+    dataset=video_minecraft_dememwm_latent \
+    "dataset.save_dir=${DATA_ROOT}" \
+    "dataset.precomputed_feature_dir=${FEATURE_ROOT}" \
+    dataset.n_frames=8 \
+    "dataset.n_frames_valid=${N_FRAMES_VALID}" \
+    "dataset.context_length=${CONTEXT_LENGTH}" \
+    dataset.single_eval_clip=true \
+    dataset.memory_selection.enabled=true \
+    dataset.memory_selection.max_anchor_frames=2 \
+    dataset.memory_selection.max_dynamic_frames=4 \
+    dataset.memory_selection.max_revisit_frames=2 \
+    dataset.memory_selection.causal=true \
+    dataset.memory_selection.dynamic.selection_policy=multiview \
+    dataset.memory_selection.dynamic.multiview_selector=fov_greedy \
+    dataset.memory_selection.pose_similarity_threshold=0.6 \
+    dataset.memory_selection.training_use_plucker=true \
+    dataset.memory_selection.training_plucker_weight=1.0 \
+    dataset.memory_selection.fov_overlap_threshold=0.6 \
+    dataset.memory_selection.min_total_selected_coverage=0.1 \
+    dataset.memory_selection.local_context_exclusion_frames=8 \
+    dataset.memory_selection.anchor_diverse_selection=true \
+    dataset.memory_selection.pose_preselect_topk=64 \
+    dataset.memory_selection.candidate_chunk_size=64 \
+    algorithm.causal=true \
+    algorithm.n_tokens=8 \
+    "algorithm.context_frames=${ALGORITHM_CONTEXT_FRAMES}" \
+    algorithm.metric_report_segment=100 \
+    +algorithm.log_per_frame_metrics=true \
+    algorithm.chunk_size=1 \
+    algorithm.warmup_steps=5000 \
+    algorithm.trainability.train_full_dit=true \
+    algorithm.trainability.full_dit_start_step=40000 \
+    algorithm.trainability.lr.memory_modules=4.0e-5 \
+    algorithm.trainability.lr.base_dit=1.0e-5 \
+    algorithm.log_video=true \
+    algorithm.save_local=true \
+    algorithm.diffusion.sampling_timesteps=20 \
+    algorithm.noise_route.anchor=all \
+    algorithm.noise_route.dynamic=all \
+    algorithm.noise_route.revisit=all \
+    algorithm.memory_noise.enabled=false \
+    algorithm.memory_noise.anchor_max_fraction=0.0 \
+    algorithm.memory_noise.dynamic_max_fraction=0.0 \
+    algorithm.memory_noise.revisit_max_fraction=0.0 \
+    +algorithm.memory_noise.validation_noisy_memory=false \
+    "experiment.training.curriculum=${CURRICULUM}" \
+    "experiment.training.batch_size=${TRAIN_BATCH_SIZE}" \
+    experiment.training.precision=16-mixed \
+    "experiment.training.optim.accumulate_grad_batches=${ACCUMULATE_GRAD_BATCHES}" \
+    experiment.training.optim.gradient_clip_val=1.0 \
+    "experiment.training.data.num_workers=${NUM_WORKERS}" \
+    "experiment.validation.data.num_workers=${NUM_WORKERS}" \
+    "experiment.validation.batch_size=${VAL_BATCH_SIZE}" \
+    "experiment.validation.limit_batch=${LIMIT_BATCH}" \
+    "experiment.validation.val_every_n_step=${VAL_EVERY_N_STEP}" \
+    "experiment.training.checkpointing.every_n_train_steps=${CHECKPOINT_EVERY_N_STEPS}" \
+    +experiment.training.checkpointing.save_top_k=5 \
+    +experiment.training.checkpointing.monitor=lpips \
+    +experiment.training.checkpointing.mode=min \
+    "experiment.training.max_steps=${MAX_STEPS}"