Upload folder using huggingface_hub

bimamba_stage_b_joint_ckpt_40k/wandb/debug.log

@@ -1,26 +1,26 @@
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Configure stats pid to 409635
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
-2026-04-12 19:23:25,279 WARNING MainThread:409635 [wandb_setup.py:_flush():77] Could not find program at -m main
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260412_192325-stage_b_joint_offline/logs/debug.log
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260412_192325-stage_b_joint_offline/logs/debug-internal.log
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:init():608] calling init triggers
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
-config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 8, 'max_epochs': -1, 'max_steps': 175000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 8, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 100, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_training', 'stage_b_joint_training': True, 'stage_b_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': True, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [50, 100, 200], '_name': 'df_video_mamba3stage', 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': 'stage_b_joint_offline', 'load': None, 'name': 'train_stage_b_mamba_joint', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/'}
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:init():658] starting backend
-2026-04-12 19:23:25,279 INFO    MainThread:409635 [wandb_init.py:init():662] setting up manager
-2026-04-12 19:23:25,301 INFO    MainThread:409635 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2026-04-12 19:23:25,309 INFO    MainThread:409635 [wandb_init.py:init():670] backend started and connected
-2026-04-12 19:23:25,334 INFO    MainThread:409635 [wandb_init.py:init():768] updated telemetry
-2026-04-12 19:23:25,403 INFO    MainThread:409635 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
-2026-04-12 19:23:25,406 INFO    MainThread:409635 [wandb_init.py:init():852] starting run threads in backend
-2026-04-12 19:23:33,168 INFO    MainThread:409635 [wandb_run.py:_console_start():2465] atexit reg
-2026-04-12 19:23:33,169 INFO    MainThread:409635 [wandb_run.py:_redirect():2311] redirect: wrap_raw
-2026-04-12 19:23:33,169 INFO    MainThread:409635 [wandb_run.py:_redirect():2376] Wrapping output streams.
-2026-04-12 19:23:33,169 INFO    MainThread:409635 [wandb_run.py:_redirect():2401] Redirects installed.
-2026-04-12 19:23:33,182 INFO    MainThread:409635 [wandb_init.py:init():895] run started, returning control to user process
-2026-04-15 04:37:36,308 WARNING MsgRouterThr:409635 [router.py:message_loop():77] message_loop has been closed
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Configure stats pid to 3907043
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-04-15 13:32:29,959 WARNING MainThread:3907043 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug.log
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug-internal.log
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():608] calling init triggers
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 8, 'max_epochs': -1, 'max_steps': 175000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 8, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 100, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_training', 'stage_b_joint_training': True, 'stage_b_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_summary_dim': 1024, 'mamba_model_dim': 1024, 'mamba_depth': 4, 'mamba_cond_dim': 1024, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': True, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [50, 100, 200], '_name': 'df_video_mamba3stage', 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': 'stage_b_joint_offline', 'load': None, 'name': 'train_stage_b_mamba_joint', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/'}
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():658] starting backend
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():662] setting up manager
+2026-04-15 13:32:29,978 INFO    MainThread:3907043 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-15 13:32:29,984 INFO    MainThread:3907043 [wandb_init.py:init():670] backend started and connected
+2026-04-15 13:32:30,020 INFO    MainThread:3907043 [wandb_init.py:init():768] updated telemetry
+2026-04-15 13:32:30,086 INFO    MainThread:3907043 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-04-15 13:32:30,089 INFO    MainThread:3907043 [wandb_init.py:init():852] starting run threads in backend
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_console_start():2465] atexit reg
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-15 13:32:38,714 INFO    MainThread:3907043 [wandb_init.py:init():895] run started, returning control to user process
+2026-04-15 13:32:52,119 WARNING MsgRouterThr:3907043 [router.py:message_loop():77] message_loop has been closed

bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/files/wandb-metadata.json

@@ -0,0 +1,1228 @@
+{
+    "os": "Linux-5.14.0-611.45.1.el9_7.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-04-15T11:32:30.245614",
+    "startedAt": "2026-04-15T11:32:29.926045",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_b_mamba_joint",
+        "algorithm=df_video_mamba3stage",
+        "experiment.num_nodes=1",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=200",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.wo_updown=false",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "+dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_b_diffusion_training",
+        "algorithm.stage_b_joint_training=true",
+        "algorithm.stage_b_memory_aux_weight=0.1",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=true",
+        "algorithm.enable_memory_noise_curriculum=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=32",
+        "algorithm.diff_window_size=8",
+        "algorithm.memory_condition_length=0",
+        "algorithm.context_frames=100",
+        "+algorithm.n_tokens=8",
+        "experiment.training.lr=2e-5",
+        "experiment.training.batch_size=8",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=175000",
+        "experiment.validation.val_every_n_step=2500",
+        "resume=stage_b_joint_offline",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "02748d25aa0f9db1e2671978dbef7b00ce028e80"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node108",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1108.489669642857,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3747.675,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3267.173,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3744.959,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.49,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3544.656,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3790.699,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3773.57,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3802.009,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3801.87,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.97,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3802.113,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.145,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3693.754,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3796.613,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.358,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.004,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.388,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.40554428100586
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.5592346191406
+    }
+}

bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 19}}

bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug-internal.log

@@ -0,0 +1,142 @@
+2026-04-15 13:32:29,982 INFO    StreamThr :3910583 [internal.py:wandb_internal():85] W&B internal server running at pid: 3910583, started at: 2026-04-15 13:32:29.979444
+2026-04-15 13:32:29,984 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status
+2026-04-15 13:32:30,021 INFO    WriterThread:3910583 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/run-stage_b_joint_offline.wandb
+2026-04-15 13:32:30,154 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: run_start
+2026-04-15 13:32:30,189 DEBUG   HandlerThread:3910583 [system_info.py:__init__():26] System info init
+2026-04-15 13:32:30,189 DEBUG   HandlerThread:3910583 [system_info.py:__init__():41] System info init done
+2026-04-15 13:32:30,189 INFO    HandlerThread:3910583 [system_monitor.py:start():194] Starting system monitor
+2026-04-15 13:32:30,189 INFO    SystemMonitor:3910583 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-04-15 13:32:30,189 INFO    HandlerThread:3910583 [system_monitor.py:probe():214] Collecting system info
+2026-04-15 13:32:30,190 INFO    SystemMonitor:3910583 [interfaces.py:start():188] Started cpu monitoring
+2026-04-15 13:32:30,191 INFO    SystemMonitor:3910583 [interfaces.py:start():188] Started disk monitoring
+2026-04-15 13:32:30,191 INFO    SystemMonitor:3910583 [interfaces.py:start():188] Started gpu monitoring
+2026-04-15 13:32:30,191 INFO    SystemMonitor:3910583 [interfaces.py:start():188] Started memory monitoring
+2026-04-15 13:32:30,192 INFO    SystemMonitor:3910583 [interfaces.py:start():188] Started network monitoring
+2026-04-15 13:32:30,245 DEBUG   HandlerThread:3910583 [system_info.py:probe():152] Probing system
+2026-04-15 13:32:30,247 DEBUG   HandlerThread:3910583 [system_info.py:_probe_git():137] Probing git
+2026-04-15 13:32:30,310 DEBUG   HandlerThread:3910583 [system_info.py:_probe_git():145] Probing git done
+2026-04-15 13:32:30,310 DEBUG   HandlerThread:3910583 [system_info.py:probe():200] Probing system done
+2026-04-15 13:32:30,310 DEBUG   HandlerThread:3910583 [system_monitor.py:probe():223] {'os': 'Linux-5.14.0-611.45.1.el9_7.x86_64-x86_64-with-glibc2.34', 'python': '3.10.19', 'heartbeatAt': '2026-04-15T11:32:30.245614', 'startedAt': '2026-04-15T11:32:29.926045', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_b_mamba_joint', 'algorithm=df_video_mamba3stage', 'experiment.num_nodes=1', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'dataset.n_frames=200', '+dataset.n_frames_valid=200', '+dataset.angle_range=110', '+dataset.pos_range=8', '+dataset.wo_updown=false', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', '+dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_b_diffusion_training', 'algorithm.stage_b_joint_training=true', 'algorithm.stage_b_memory_aux_weight=0.1', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=true', 'algorithm.enable_memory_noise_curriculum=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=32', 'algorithm.diff_window_size=8', 'algorithm.memory_condition_length=0', 'algorithm.context_frames=100', '+algorithm.n_tokens=8', 'experiment.training.lr=2e-5', 'experiment.training.batch_size=8', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=175000', 'experiment.validation.val_every_n_step=2500', 'resume=stage_b_joint_offline', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': '02748d25aa0f9db1e2671978dbef7b00ce028e80'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node108', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 1108.489669642857, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 3747.675, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.038, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.038, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3267.173, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3744.959, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3798.49, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3544.656, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3790.699, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3773.57, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3802.009, 'min': 800.0, 'max': 3800.0}, {'current': 3801.87, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3797.97, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3802.113, 'min': 800.0, 'max': 3800.0}, {'current': 3800.145, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3693.754, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3796.613, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.358, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.961, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.961, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3804.004, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3797.388, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 30.58770751953125, 'used': 16.40554428100586}}, 'gpu': 'NVIDIA H200', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}], 'memory': {'total': 2015.5592346191406}}
+2026-04-15 13:32:30,310 INFO    HandlerThread:3910583 [system_monitor.py:probe():224] Finished collecting system info
+2026-04-15 13:32:30,310 INFO    HandlerThread:3910583 [system_monitor.py:probe():227] Publishing system info
+2026-04-15 13:32:30,311 DEBUG   HandlerThread:3910583 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-04-15 13:32:36,797 DEBUG   HandlerThread:3910583 [system_info.py:_save_conda():224] Saving conda packages done
+2026-04-15 13:32:36,804 INFO    HandlerThread:3910583 [system_monitor.py:probe():229] Finished publishing system info
+2026-04-15 13:32:36,820 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: keepalive
+2026-04-15 13:32:36,820 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status_report
+2026-04-15 13:32:36,820 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: status_report
+2026-04-15 13:32:38,704 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: python_packages
+2026-04-15 13:32:40,821 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status_report
+2026-04-15 13:32:40,822 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: status_report
+2026-04-15 13:32:45,823 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status_report
+2026-04-15 13:32:45,823 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: status_report
+2026-04-15 13:32:49,979 DEBUG   SenderThread:3910583 [sender.py:send():391] send: exit
+2026-04-15 13:32:49,979 INFO    SenderThread:3910583 [sender.py:send_exit():598] handling exit code: 1
+2026-04-15 13:32:49,979 INFO    SenderThread:3910583 [sender.py:send_exit():600] handling runtime: 19
+2026-04-15 13:32:49,983 INFO    SenderThread:3910583 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2026-04-15 13:32:49,983 INFO    SenderThread:3910583 [sender.py:send_exit():606] send defer
+2026-04-15 13:32:49,984 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:49,984 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 0
+2026-04-15 13:32:49,984 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:49,984 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 0
+2026-04-15 13:32:49,984 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 1
+2026-04-15 13:32:49,984 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:49,984 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 1
+2026-04-15 13:32:49,984 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:49,984 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 1
+2026-04-15 13:32:49,985 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 2
+2026-04-15 13:32:49,985 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:49,985 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 2
+2026-04-15 13:32:49,985 INFO    HandlerThread:3910583 [system_monitor.py:finish():203] Stopping system monitor
+2026-04-15 13:32:49,986 DEBUG   SystemMonitor:3910583 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-04-15 13:32:49,986 DEBUG   SystemMonitor:3910583 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2026-04-15 13:32:49,986 DEBUG   SystemMonitor:3910583 [system_monitor.py:_start():183] Publishing last batch of metrics
+2026-04-15 13:32:49,987 INFO    HandlerThread:3910583 [interfaces.py:finish():200] Joined cpu monitor
+2026-04-15 13:32:49,987 INFO    HandlerThread:3910583 [interfaces.py:finish():200] Joined disk monitor
+2026-04-15 13:32:51,097 INFO    HandlerThread:3910583 [interfaces.py:finish():200] Joined gpu monitor
+2026-04-15 13:32:51,098 INFO    HandlerThread:3910583 [interfaces.py:finish():200] Joined memory monitor
+2026-04-15 13:32:51,098 INFO    HandlerThread:3910583 [interfaces.py:finish():200] Joined network monitor
+2026-04-15 13:32:51,098 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: poll_exit
+2026-04-15 13:32:51,098 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status_report
+2026-04-15 13:32:51,099 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,099 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 2
+2026-04-15 13:32:51,099 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 3
+2026-04-15 13:32:51,099 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: poll_exit
+2026-04-15 13:32:51,099 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: status_report
+2026-04-15 13:32:51,099 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,099 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 3
+2026-04-15 13:32:51,099 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,099 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 3
+2026-04-15 13:32:51,099 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 4
+2026-04-15 13:32:51,100 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,100 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 4
+2026-04-15 13:32:51,100 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,100 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 4
+2026-04-15 13:32:51,100 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 5
+2026-04-15 13:32:51,100 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,100 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 5
+2026-04-15 13:32:51,101 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,101 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 5
+2026-04-15 13:32:51,101 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 6
+2026-04-15 13:32:51,101 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,101 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 6
+2026-04-15 13:32:51,102 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,102 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 6
+2026-04-15 13:32:51,102 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 7
+2026-04-15 13:32:51,102 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: status_report
+2026-04-15 13:32:51,102 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,102 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 7
+2026-04-15 13:32:51,102 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: status_report
+2026-04-15 13:32:51,102 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,102 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 7
+2026-04-15 13:32:51,102 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 8
+2026-04-15 13:32:51,103 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,103 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 8
+2026-04-15 13:32:51,103 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 8
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 9
+2026-04-15 13:32:51,103 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,103 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 9
+2026-04-15 13:32:51,103 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 9
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 10
+2026-04-15 13:32:51,103 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,103 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 10
+2026-04-15 13:32:51,103 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 10
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 11
+2026-04-15 13:32:51,103 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,103 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 11
+2026-04-15 13:32:51,103 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 11
+2026-04-15 13:32:51,103 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 12
+2026-04-15 13:32:51,103 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,103 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 12
+2026-04-15 13:32:51,103 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,104 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 12
+2026-04-15 13:32:51,104 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 13
+2026-04-15 13:32:51,104 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,104 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 13
+2026-04-15 13:32:51,104 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,104 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 13
+2026-04-15 13:32:51,104 INFO    SenderThread:3910583 [sender.py:transition_state():626] send defer: 14
+2026-04-15 13:32:51,104 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: defer
+2026-04-15 13:32:51,104 INFO    HandlerThread:3910583 [handler.py:handle_request_defer():184] handle defer: 14
+2026-04-15 13:32:51,104 DEBUG   SenderThread:3910583 [sender.py:send():391] send: final
+2026-04-15 13:32:51,104 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: defer
+2026-04-15 13:32:51,104 INFO    SenderThread:3910583 [sender.py:send_request_defer():622] handle sender defer: 14
+2026-04-15 13:32:51,105 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: poll_exit
+2026-04-15 13:32:51,105 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: poll_exit
+2026-04-15 13:32:51,105 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: server_info
+2026-04-15 13:32:51,105 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: server_info
+2026-04-15 13:32:51,106 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: poll_exit
+2026-04-15 13:32:51,106 DEBUG   SenderThread:3910583 [sender.py:send_request():418] send_request: poll_exit
+2026-04-15 13:32:51,106 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: get_summary
+2026-04-15 13:32:51,106 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: sampled_history
+2026-04-15 13:32:51,107 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: internal_messages
+2026-04-15 13:32:51,107 INFO    MainThread:3910583 [wandb_run.py:_footer_history_summary_info():4088] rendering history
+2026-04-15 13:32:51,107 INFO    MainThread:3910583 [wandb_run.py:_footer_history_summary_info():4120] rendering summary
+2026-04-15 13:32:51,107 DEBUG   HandlerThread:3910583 [handler.py:handle_request():158] handle_request: shutdown
+2026-04-15 13:32:51,107 INFO    HandlerThread:3910583 [handler.py:finish():884] shutting down handler
+2026-04-15 13:32:52,106 INFO    WriterThread:3910583 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/run-stage_b_joint_offline.wandb
+2026-04-15 13:32:52,106 INFO    SenderThread:3910583 [sender.py:finish():1631] shutting down sender

bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Configure stats pid to 3907043
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-04-15 13:32:29,959 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-04-15 13:32:29,959 WARNING MainThread:3907043 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug.log
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/wandb/offline-run-20260415_133229-stage_b_joint_offline/logs/debug-internal.log
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():608] calling init triggers
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 8, 'max_epochs': -1, 'max_steps': 175000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 8, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 100, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_training', 'stage_b_joint_training': True, 'stage_b_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_summary_dim': 1024, 'mamba_model_dim': 1024, 'mamba_depth': 4, 'mamba_cond_dim': 1024, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': True, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [50, 100, 200], '_name': 'df_video_mamba3stage', 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': 'stage_b_joint_offline', 'load': None, 'name': 'train_stage_b_mamba_joint', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b_joint_ckpt_40k/'}
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():658] starting backend
+2026-04-15 13:32:29,960 INFO    MainThread:3907043 [wandb_init.py:init():662] setting up manager
+2026-04-15 13:32:29,978 INFO    MainThread:3907043 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-15 13:32:29,984 INFO    MainThread:3907043 [wandb_init.py:init():670] backend started and connected
+2026-04-15 13:32:30,020 INFO    MainThread:3907043 [wandb_init.py:init():768] updated telemetry
+2026-04-15 13:32:30,086 INFO    MainThread:3907043 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-04-15 13:32:30,089 INFO    MainThread:3907043 [wandb_init.py:init():852] starting run threads in backend
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_console_start():2465] atexit reg
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-15 13:32:38,704 INFO    MainThread:3907043 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-15 13:32:38,714 INFO    MainThread:3907043 [wandb_init.py:init():895] run started, returning control to user process
+2026-04-15 13:32:52,119 WARNING MsgRouterThr:3907043 [router.py:message_loop():77] message_loop has been closed