Upload folder using huggingface_hub

.gitattributes

@@ -4010,3 +4010,9 @@ dememwm-curric-event-anchorclean-causal-route-safe-nodynamic-8h200-35034.out fil
 train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/media/videos/validation_vis/batch000000_0_0_e955857262f58b0db142.mp4 filter=lfs diff=lfs merge=lfs -text
 train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/files/media/videos/validation_vis/batch000000_0_83_ca560e3c69f3131c07fa.mp4 filter=lfs diff=lfs merge=lfs -text
 train_dememwm_unified_dense_memory_base_8h200_bs16_global128_120k/wandb/offline-run-20260710_111005-qv85iql0/run-qv85iql0.wandb filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_0_81725955c6da72ad2403.mp4 filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_166_64bb6ae0f25422172d0f.mp4 filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_249_6989ce1d0de9038c8887.mp4 filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_332_72e91212a833db992486.mp4 filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_83_c1afdaeec94523e72f11.mp4 filter=lfs diff=lfs merge=lfs -text
+train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/run-8vnil88b.wandb filter=lfs diff=lfs merge=lfs -text

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step114000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06ca7e1f44c0afcde1e36ea26f7c89dc3d1a2829703dcd7b0236396ca8237af3
+size 13466697166

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step116000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7aee7e668d3df0664ea5d8bced7a782bd52319cb6bbf75a7b6117f9517ef668e
+size 13466697166

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step118000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b09bf6bc71102e552ffc91ca72e5f40b2745437d794666403d6abcd8d7638be6
+size 13466697166

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step120000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e269d0c8ad9ac4bdb65000a1e79fedaf24d0cf26e955818d016ada2428fd89d8
+size 13466697166

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/debug.log

@@ -1,26 +1,26 @@
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Configure stats pid to 3328795
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/DeMemWM/wandb/settings
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'api_key': '***REDACTED***', 'mode': 'dryrun'}
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
-2026-07-06 21:36:25,633 WARNING MainThread:3328795 [wandb_setup.py:_flush():77] Could not find program at -m main
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/logs/debug.log
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260706_213625-8vnil88b/logs/debug-internal.log
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:init():608] calling init triggers
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
-config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 16, 'max_epochs': -1, 'max_steps': 120000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2000, 'every_n_epochs': None, 'train_time_interval': None, 'save_last_k': 5, 'enable_version_counter': False}, 'curriculum': {'enabled': True, 'stages': [{'name': 'no_updown', 'until_step': 40000, 'dataset': {'wo_updown': True}}, {'name': 'with_updown', 'until_step': 120000, 'dataset': {'wo_updown': False}}]}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'val_every_n_step': 2000, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'metadata': None, 'n_frames': 8, 'n_frames_valid': 1100, 'frame_skip': 1, 'resolution': 128, 'image_hw': [360, 640], 'observation_shape': [16, 18, 32], 'data_mean': 0.0, 'data_std': 1.0, 'action_cond_dim': 25, 'context_length': 100, 'single_eval_clip': True, 'shuffle_clips': True, 'memory_selection': {'enabled': True, 'causal': True, 'max_anchor_frames': 2, 'max_dynamic_frames': 4, 'max_revisit_frames': 2, 'pose_similarity_threshold': 0.6, 'training_use_plucker': True, 'training_plucker_weight': 1.0, 'training_dropout': 0.1, 'fov_overlap_threshold': 0.6, 'min_total_selected_coverage': 0.1, 'local_context_exclusion_frames': 8, 'plucker_moment_radius': 30.0, 'anchor_diverse_selection': True, 'pose_preselect_topk': 32, 'candidate_chunk_size': 0, 'dynamic': {'selection_policy': 'multiview', 'multiview_selector': 'fov_greedy', 'scene_threshold': 2.5, 'state_threshold': 2.5, 'stable_threshold': 1.0, 'stable_frames': 3, 'min_event_gap': 8, 'min_anchor_score': 2.0, 'max_event_anchors': None, 'b_pose': 0.3, 'b_action': 0.2}}, '_name': 'video_minecraft_dememwm_latent'}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 600, 'weight_decay': 0.002, 'warmup_steps': 500, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'memory_selection': '${dataset.memory_selection}', 'memory_noise': {'enabled': True, 'mode': 'random_cleaner_fraction', 'anchor_max_fraction': 0.25, 'dynamic_max_fraction': 0.5, 'revisit_max_fraction': 0.5, 'validation_noisy_memory': False}, 'noise_route': {'anchor': 'high', 'dynamic': 'low', 'revisit': 'all'}, 'trainability': {'freeze_vae': True, 'reference_attention': True, 'adaln_mlp': True, 'geometry_projections': True, 'train_full_dit': True, 'full_dit_start_step': 10000, 'lr': {'memory_modules': 8e-05, 'base_dit': 2e-05}}, 'n_tokens': 8, 'action_cond_dim': '${dataset.action_cond_dim}', 'pose_cond_dim': 5, 'use_plucker': True, 'relative_embedding': True, 'state_embed_only_on_qk': True, 'use_memory_attention': True, 'memory_attention_key_only_geometry': True, 'add_timestamp_embedding': False, 'ref_mode': 'sequential', 'focal_length': 0.35, 'log_video': True, 'save_local': False, 'metric_report_segment': 100, 'require_pose_prediction': False, '_name': 'dememwm_base'}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'offline'}, 'resume': None, 'load': None, 'auto_resume': True, 'resume_ckpt_path': None, 'name': 'train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k', 'output_dir': '/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors', '_auto_resuming': False, '_resume_checkpoint_path': None}
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:init():658] starting backend
-2026-07-06 21:36:25,633 INFO    MainThread:3328795 [wandb_init.py:init():662] setting up manager
-2026-07-06 21:36:25,660 INFO    MainThread:3328795 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2026-07-06 21:36:25,670 INFO    MainThread:3328795 [wandb_init.py:init():670] backend started and connected
-2026-07-06 21:36:25,700 INFO    MainThread:3328795 [wandb_init.py:init():768] updated telemetry
-2026-07-06 21:36:25,765 INFO    MainThread:3328795 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
-2026-07-06 21:36:25,774 INFO    MainThread:3328795 [wandb_init.py:init():852] starting run threads in backend
-2026-07-06 21:36:37,474 INFO    MainThread:3328795 [wandb_run.py:_console_start():2465] atexit reg
-2026-07-06 21:36:37,475 INFO    MainThread:3328795 [wandb_run.py:_redirect():2311] redirect: wrap_raw
-2026-07-06 21:36:37,475 INFO    MainThread:3328795 [wandb_run.py:_redirect():2376] Wrapping output streams.
-2026-07-06 21:36:37,475 INFO    MainThread:3328795 [wandb_run.py:_redirect():2401] Redirects installed.
-2026-07-06 21:36:37,488 INFO    MainThread:3328795 [wandb_init.py:init():895] run started, returning control to user process
-2026-07-08 15:01:30,572 WARNING MsgRouterThr:3328795 [router.py:message_loop():77] message_loop has been closed
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Configure stats pid to 2769672
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/DeMemWM/wandb/settings
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'api_key': '***REDACTED***', 'mode': 'dryrun'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-07-10 11:08:38,424 WARNING MainThread:2769672 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/logs/debug.log
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/logs/debug-internal.log
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():608] calling init triggers
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 16, 'max_epochs': -1, 'max_steps': 120000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2000, 'every_n_epochs': None, 'train_time_interval': None, 'save_last_k': 5, 'enable_version_counter': False}, 'curriculum': {'enabled': True, 'stages': [{'name': 'no_updown', 'until_step': 40000, 'dataset': {'wo_updown': True}}, {'name': 'with_updown', 'until_step': 120000, 'dataset': {'wo_updown': False}}]}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'val_every_n_step': 2000, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'metadata': None, 'n_frames': 8, 'n_frames_valid': 1100, 'frame_skip': 1, 'resolution': 128, 'image_hw': [360, 640], 'observation_shape': [16, 18, 32], 'data_mean': 0.0, 'data_std': 1.0, 'action_cond_dim': 25, 'context_length': 100, 'single_eval_clip': True, 'shuffle_clips': True, 'memory_selection': {'enabled': True, 'causal': True, 'max_anchor_frames': 2, 'max_dynamic_frames': 4, 'max_revisit_frames': 2, 'pose_similarity_threshold': 0.6, 'training_use_plucker': True, 'training_plucker_weight': 1.0, 'training_dropout': 0.1, 'fov_overlap_threshold': 0.6, 'min_total_selected_coverage': 0.1, 'local_context_exclusion_frames': 8, 'plucker_moment_radius': 30.0, 'anchor_diverse_selection': True, 'pose_preselect_topk': 32, 'candidate_chunk_size': 0, 'dynamic': {'selection_policy': 'multiview', 'multiview_selector': 'fov_greedy', 'scene_threshold': 2.5, 'state_threshold': 2.5, 'stable_threshold': 1.0, 'stable_frames': 3, 'min_event_gap': 8, 'min_anchor_score': 2.0, 'max_event_anchors': None, 'b_pose': 0.3, 'b_action': 0.2}}, '_name': 'video_minecraft_dememwm_latent'}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 600, 'weight_decay': 0.002, 'warmup_steps': 500, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'memory_selection': '${dataset.memory_selection}', 'memory_noise': {'enabled': True, 'mode': 'random_cleaner_fraction', 'anchor_max_fraction': 0.25, 'dynamic_max_fraction': 0.5, 'revisit_max_fraction': 0.5, 'validation_noisy_memory': False}, 'noise_route': {'anchor': 'high', 'dynamic': 'low', 'revisit': 'all'}, 'trainability': {'freeze_vae': True, 'reference_attention': True, 'adaln_mlp': True, 'geometry_projections': True, 'train_full_dit': True, 'full_dit_start_step': 10000, 'lr': {'memory_modules': 8e-05, 'base_dit': 2e-05}}, 'n_tokens': 8, 'action_cond_dim': '${dataset.action_cond_dim}', 'pose_cond_dim': 5, 'use_plucker': True, 'relative_embedding': True, 'state_embed_only_on_qk': True, 'use_memory_attention': True, 'memory_attention_key_only_geometry': True, 'add_timestamp_embedding': False, 'ref_mode': 'sequential', 'focal_length': 0.35, 'log_video': True, 'save_local': False, 'metric_report_segment': 100, 'require_pose_prediction': False, '_name': 'dememwm_base'}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'offline'}, 'resume': None, 'load': None, 'auto_resume': True, 'resume_ckpt_path': None, 'name': 'train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k', 'output_dir': '/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors', '_auto_resuming': True, '_resume_checkpoint_path': '/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step112000.ckpt'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():658] starting backend
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():662] setting up manager
+2026-07-10 11:08:38,444 INFO    MainThread:2769672 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-07-10 11:08:38,454 INFO    MainThread:2769672 [wandb_init.py:init():670] backend started and connected
+2026-07-10 11:08:38,475 INFO    MainThread:2769672 [wandb_init.py:init():768] updated telemetry
+2026-07-10 11:08:38,526 INFO    MainThread:2769672 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-07-10 11:08:38,534 INFO    MainThread:2769672 [wandb_init.py:init():852] starting run threads in backend
+2026-07-10 11:08:50,124 INFO    MainThread:2769672 [wandb_run.py:_console_start():2465] atexit reg
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-07-10 11:08:50,141 INFO    MainThread:2769672 [wandb_init.py:init():895] run started, returning control to user process
+2026-07-10 14:19:13,987 WARNING MsgRouterThr:2769672 [router.py:message_loop():77] message_loop has been closed

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/table/segment_metrics/table_167_a147dc1537a671218d58.table.json

@@ -0,0 +1 @@
+{"columns": ["generation_start", "generation_end", "generation_length", "absolute_start", "absolute_end", "mse", "psnr", "lpips", "count"], "data": [[0, 100, 100, 600, 700, 0.023421650752425194, 20.886640548706055, 0.1506877839565277, 800.0], [100, 200, 100, 700, 800, 0.046571116894483566, 17.80234146118164, 0.22302976250648499, 800.0], [200, 300, 100, 800, 900, 0.06072252243757248, 17.788999557495117, 0.2545630931854248, 800.0], [300, 400, 100, 900, 1000, 0.051105666905641556, 15.917095184326172, 0.24307380616664886, 800.0], [400, 500, 100, 1000, 1100, 0.06658346205949783, 17.29439926147461, 0.24850650131702423, 800.0]]}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/table/segment_metrics/table_1_7d09849323174d116153.table.json

@@ -0,0 +1 @@
+{"columns": ["generation_start", "generation_end", "generation_length", "absolute_start", "absolute_end", "mse", "psnr", "lpips", "count"], "data": [[0, 100, 100, 600, 700, 0.021900638937950134, 21.31537628173828, 0.13578718900680542, 800.0], [100, 200, 100, 700, 800, 0.042063161730766296, 18.343778610229492, 0.2159149944782257, 800.0], [200, 300, 100, 800, 900, 0.04293680191040039, 18.077938079833984, 0.24056419730186462, 800.0], [300, 400, 100, 900, 1000, 0.061268072575330734, 14.972661972045898, 0.28722357749938965, 800.0], [400, 500, 100, 1000, 1100, 0.039312928915023804, 18.417510986328125, 0.24477188289165497, 800.0]]}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/table/segment_metrics/table_250_8807275564f646c67682.table.json

@@ -0,0 +1 @@
+{"columns": ["generation_start", "generation_end", "generation_length", "absolute_start", "absolute_end", "mse", "psnr", "lpips", "count"], "data": [[0, 100, 100, 600, 700, 0.025593996047973633, 20.49295425415039, 0.1694771647453308, 800.0], [100, 200, 100, 700, 800, 0.04745453968644142, 17.660537719726562, 0.2210940569639206, 800.0], [200, 300, 100, 800, 900, 0.05088909715414047, 17.630664825439453, 0.24497058987617493, 800.0], [300, 400, 100, 900, 1000, 0.061335936188697815, 15.122014045715332, 0.2563353180885315, 800.0], [400, 500, 100, 1000, 1100, 0.058958034962415695, 15.930368423461914, 0.2580588459968567, 800.0]]}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/table/segment_metrics/table_333_85e930d378d45451f4a9.table.json

@@ -0,0 +1 @@
+{"columns": ["generation_start", "generation_end", "generation_length", "absolute_start", "absolute_end", "mse", "psnr", "lpips", "count"], "data": [[0, 100, 100, 600, 700, 0.02093818038702011, 21.416858673095703, 0.14256158471107483, 800.0], [100, 200, 100, 700, 800, 0.043739404529333115, 18.049684524536133, 0.2187359631061554, 800.0], [200, 300, 100, 800, 900, 0.043406371027231216, 19.183998107910156, 0.23073707520961761, 800.0], [300, 400, 100, 900, 1000, 0.05077726021409035, 16.302196502685547, 0.22290201485157013, 800.0], [400, 500, 100, 1000, 1100, 0.06947535276412964, 17.940200805664062, 0.24432289600372314, 800.0]]}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/table/segment_metrics/table_84_e4b0158ea11eca8db267.table.json

@@ -0,0 +1 @@
+{"columns": ["generation_start", "generation_end", "generation_length", "absolute_start", "absolute_end", "mse", "psnr", "lpips", "count"], "data": [[0, 100, 100, 600, 700, 0.02209002897143364, 21.13543128967285, 0.14256349205970764, 800.0], [100, 200, 100, 700, 800, 0.04153624549508095, 18.286083221435547, 0.19715221226215363, 800.0], [200, 300, 100, 800, 900, 0.04025625064969063, 17.56735610961914, 0.21875838935375214, 800.0], [300, 400, 100, 900, 1000, 0.05111774057149887, 15.158273696899414, 0.22227683663368225, 800.0], [400, 500, 100, 1000, 1100, 0.07586982846260071, 15.380361557006836, 0.26255884766578674, 800.0]]}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_0_81725955c6da72ad2403.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81725955c6da72ad24034620c052a56e82e37c0b0f4295e4501f464fde7ebe01
+size 9107843

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_166_64bb6ae0f25422172d0f.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64bb6ae0f25422172d0f3fc7b04e1eb59fcf2ac854828218c7a9d92f5e09de4b
+size 8935524

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_249_6989ce1d0de9038c8887.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6989ce1d0de9038c8887d9f8471544320b2eba3157cfd494fb6f9b53f7e9cda2
+size 9425144

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_332_72e91212a833db992486.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72e91212a833db99248652a5a6e6ea55e518520987a0a055ce3cf47d9a01608a
+size 8681816

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/media/videos/validation_vis/batch000000_0_83_c1afdaeec94523e72f11.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1afdaeec94523e72f11a7a59b7a7c84d0deca05004a39b879c1f1cdaa62447f
+size 9123707

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00112000.csv

@@ -0,0 +1,6 @@
+generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count
+0,100,100,600,700,0.021900638937950134,21.31537628173828,0.13578718900680542,800.0
+100,200,100,700,800,0.042063161730766296,18.343778610229492,0.2159149944782257,800.0
+200,300,100,800,900,0.04293680191040039,18.077938079833984,0.24056419730186462,800.0
+300,400,100,900,1000,0.061268072575330734,14.972661972045898,0.28722357749938965,800.0
+400,500,100,1000,1100,0.039312928915023804,18.417510986328125,0.24477188289165497,800.0

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00114000.csv

@@ -0,0 +1,6 @@
+generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count
+0,100,100,600,700,0.02209002897143364,21.13543128967285,0.14256349205970764,800.0
+100,200,100,700,800,0.04153624549508095,18.286083221435547,0.19715221226215363,800.0
+200,300,100,800,900,0.04025625064969063,17.56735610961914,0.21875838935375214,800.0
+300,400,100,900,1000,0.05111774057149887,15.158273696899414,0.22227683663368225,800.0
+400,500,100,1000,1100,0.07586982846260071,15.380361557006836,0.26255884766578674,800.0

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00116000.csv

@@ -0,0 +1,6 @@
+generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count
+0,100,100,600,700,0.023421650752425194,20.886640548706055,0.1506877839565277,800.0
+100,200,100,700,800,0.046571116894483566,17.80234146118164,0.22302976250648499,800.0
+200,300,100,800,900,0.06072252243757248,17.788999557495117,0.2545630931854248,800.0
+300,400,100,900,1000,0.051105666905641556,15.917095184326172,0.24307380616664886,800.0
+400,500,100,1000,1100,0.06658346205949783,17.29439926147461,0.24850650131702423,800.0

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00118000.csv

@@ -0,0 +1,6 @@
+generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count
+0,100,100,600,700,0.025593996047973633,20.49295425415039,0.1694771647453308,800.0
+100,200,100,700,800,0.04745453968644142,17.660537719726562,0.2210940569639206,800.0
+200,300,100,800,900,0.05088909715414047,17.630664825439453,0.24497058987617493,800.0
+300,400,100,900,1000,0.061335936188697815,15.122014045715332,0.2563353180885315,800.0
+400,500,100,1000,1100,0.058958034962415695,15.930368423461914,0.2580588459968567,800.0

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/metrics/segment_metrics_step00120000.csv

@@ -0,0 +1,6 @@
+generation_start,generation_end,generation_length,absolute_start,absolute_end,mse,psnr,lpips,count
+0,100,100,600,700,0.02093818038702011,21.416858673095703,0.14256158471107483,800.0
+100,200,100,700,800,0.043739404529333115,18.049684524536133,0.2187359631061554,800.0
+200,300,100,800,900,0.043406371027231216,19.183998107910156,0.23073707520961761,800.0
+300,400,100,900,1000,0.05077726021409035,16.302196502685547,0.22290201485157013,800.0
+400,500,100,1000,1100,0.06947535276412964,17.940200805664062,0.24432289600372314,800.0

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/wandb-metadata.json

@@ -0,0 +1,1250 @@
+{
+    "os": "Linux-5.14.0-687.12.1.el9_8.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-07-10T09:08:38.708709",
+    "startedAt": "2026-07-10T09:08:38.397452",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k",
+        "+output_dir=/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/",
+        "auto_resume=true",
+        "wandb.mode=offline",
+        "experiment.tasks=[training]",
+        "algorithm=dememwm_base",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors",
+        "dataset=video_minecraft_dememwm_latent",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.n_frames=8",
+        "dataset.n_frames_valid=1100",
+        "dataset.context_length=100",
+        "dataset.single_eval_clip=true",
+        "dataset.memory_selection.enabled=true",
+        "dataset.memory_selection.causal=true",
+        "dataset.memory_selection.max_anchor_frames=2",
+        "dataset.memory_selection.max_dynamic_frames=4",
+        "dataset.memory_selection.max_revisit_frames=2",
+        "dataset.memory_selection.dynamic.selection_policy=multiview",
+        "dataset.memory_selection.dynamic.multiview_selector=fov_greedy",
+        "algorithm.n_tokens=8",
+        "algorithm.context_frames=600",
+        "algorithm.metric_report_segment=100",
+        "algorithm.memory_attention_key_only_geometry=true",
+        "algorithm.add_timestamp_embedding=false",
+        "algorithm.chunk_size=1",
+        "algorithm.warmup_steps=500",
+        "algorithm.trainability.train_full_dit=true",
+        "algorithm.trainability.full_dit_start_step=10000",
+        "algorithm.trainability.lr.memory_modules=8.0e-5",
+        "algorithm.trainability.lr.base_dit=2.0e-5",
+        "algorithm.log_video=true",
+        "algorithm.save_local=false",
+        "algorithm.diffusion.sampling_timesteps=20",
+        "algorithm.noise_route.anchor=high",
+        "algorithm.noise_route.dynamic=low",
+        "algorithm.noise_route.revisit=all",
+        "algorithm.memory_noise.enabled=true",
+        "algorithm.memory_noise.mode=random_cleaner_fraction",
+        "algorithm.memory_noise.anchor_max_fraction=0.25",
+        "algorithm.memory_noise.dynamic_max_fraction=0.50",
+        "algorithm.memory_noise.revisit_max_fraction=0.50",
+        "experiment.training.curriculum={enabled:true,stages:[{name:no_updown,until_step:40000,dataset:{wo_updown:true}},{name:with_updown,until_step:120000,dataset:{wo_updown:false}}]}",
+        "experiment.training.batch_size=16",
+        "experiment.validation.batch_size=1",
+        "experiment.validation.limit_batch=1",
+        "experiment.training.checkpointing.every_n_train_steps=2000",
+        "experiment.training.checkpointing.save_last_k=5",
+        "experiment.validation.val_every_n_step=2000",
+        "experiment.training.max_steps=120000"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/DeMemWM",
+        "commit": "c65e3e6dc1b0f3a9f5f0f4867aa5778bc05d2914"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/DeMemWM",
+    "host": "node112",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1058.444169642857,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3780.374,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3801.135,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3313.508,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3792.826,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.178,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3795.605,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3796.589,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3779.907,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3295.753,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3281.075,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.757,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.35,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.23,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.887,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.889190673828125
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.559413909912
+    }
+}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 11433}}

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/logs/debug.log

@@ -0,0 +1,26 @@
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Configure stats pid to 2769672
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-07-10 11:08:38,423 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/DeMemWM/wandb/settings
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'api_key': '***REDACTED***', 'mode': 'dryrun'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-07-10 11:08:38,424 WARNING MainThread:2769672 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/logs/debug.log
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/logs/debug-internal.log
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():608] calling init triggers
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 16, 'max_epochs': -1, 'max_steps': 120000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2000, 'every_n_epochs': None, 'train_time_interval': None, 'save_last_k': 5, 'enable_version_counter': False}, 'curriculum': {'enabled': True, 'stages': [{'name': 'no_updown', 'until_step': 40000, 'dataset': {'wo_updown': True}}, {'name': 'with_updown', 'until_step': 120000, 'dataset': {'wo_updown': False}}]}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'val_every_n_step': 2000, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'metadata': None, 'n_frames': 8, 'n_frames_valid': 1100, 'frame_skip': 1, 'resolution': 128, 'image_hw': [360, 640], 'observation_shape': [16, 18, 32], 'data_mean': 0.0, 'data_std': 1.0, 'action_cond_dim': 25, 'context_length': 100, 'single_eval_clip': True, 'shuffle_clips': True, 'memory_selection': {'enabled': True, 'causal': True, 'max_anchor_frames': 2, 'max_dynamic_frames': 4, 'max_revisit_frames': 2, 'pose_similarity_threshold': 0.6, 'training_use_plucker': True, 'training_plucker_weight': 1.0, 'training_dropout': 0.1, 'fov_overlap_threshold': 0.6, 'min_total_selected_coverage': 0.1, 'local_context_exclusion_frames': 8, 'plucker_moment_radius': 30.0, 'anchor_diverse_selection': True, 'pose_preselect_topk': 32, 'candidate_chunk_size': 0, 'dynamic': {'selection_policy': 'multiview', 'multiview_selector': 'fov_greedy', 'scene_threshold': 2.5, 'state_threshold': 2.5, 'stable_threshold': 1.0, 'stable_frames': 3, 'min_event_gap': 8, 'min_anchor_score': 2.0, 'max_event_anchors': None, 'b_pose': 0.3, 'b_action': 0.2}}, '_name': 'video_minecraft_dememwm_latent'}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 600, 'weight_decay': 0.002, 'warmup_steps': 500, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'memory_selection': '${dataset.memory_selection}', 'memory_noise': {'enabled': True, 'mode': 'random_cleaner_fraction', 'anchor_max_fraction': 0.25, 'dynamic_max_fraction': 0.5, 'revisit_max_fraction': 0.5, 'validation_noisy_memory': False}, 'noise_route': {'anchor': 'high', 'dynamic': 'low', 'revisit': 'all'}, 'trainability': {'freeze_vae': True, 'reference_attention': True, 'adaln_mlp': True, 'geometry_projections': True, 'train_full_dit': True, 'full_dit_start_step': 10000, 'lr': {'memory_modules': 8e-05, 'base_dit': 2e-05}}, 'n_tokens': 8, 'action_cond_dim': '${dataset.action_cond_dim}', 'pose_cond_dim': 5, 'use_plucker': True, 'relative_embedding': True, 'state_embed_only_on_qk': True, 'use_memory_attention': True, 'memory_attention_key_only_geometry': True, 'add_timestamp_embedding': False, 'ref_mode': 'sequential', 'focal_length': 0.35, 'log_video': True, 'save_local': False, 'metric_report_segment': 100, 'require_pose_prediction': False, '_name': 'dememwm_base'}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'offline'}, 'resume': None, 'load': None, 'auto_resume': True, 'resume_ckpt_path': None, 'name': 'train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k', 'output_dir': '/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/oasis500m.safetensors', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vit-l-20.safetensors', '_auto_resuming': True, '_resume_checkpoint_path': '/proj/cvl/users/x_fahkh2/DeMemWM/results/train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/checkpoints/epoch2_step112000.ckpt'}
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():658] starting backend
+2026-07-10 11:08:38,424 INFO    MainThread:2769672 [wandb_init.py:init():662] setting up manager
+2026-07-10 11:08:38,444 INFO    MainThread:2769672 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-07-10 11:08:38,454 INFO    MainThread:2769672 [wandb_init.py:init():670] backend started and connected
+2026-07-10 11:08:38,475 INFO    MainThread:2769672 [wandb_init.py:init():768] updated telemetry
+2026-07-10 11:08:38,526 INFO    MainThread:2769672 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-07-10 11:08:38,534 INFO    MainThread:2769672 [wandb_init.py:init():852] starting run threads in backend
+2026-07-10 11:08:50,124 INFO    MainThread:2769672 [wandb_run.py:_console_start():2465] atexit reg
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-07-10 11:08:50,125 INFO    MainThread:2769672 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-07-10 11:08:50,141 INFO    MainThread:2769672 [wandb_init.py:init():895] run started, returning control to user process
+2026-07-10 14:19:13,987 WARNING MsgRouterThr:2769672 [router.py:message_loop():77] message_loop has been closed

train_dememwm_curric_multiview_causal_a2d4r2_route_ntok8_8h200_bs16_global128_120k/wandb/offline-run-20260710_110838-8vnil88b/run-8vnil88b.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac54e80a256302445d534837816e672939ea4b54161aa1cd23c9024e41d3fe5d
+size 8548550