Upload folder using huggingface_hub

.gitattributes

@@ -124,3 +124,6 @@ wandb/offline-run-20260212_223956-ot7jqmgn/run-ot7jqmgn.wandb filter=lfs diff=lf
 slurmm1.0-15521365.out filter=lfs diff=lfs merge=lfs -text
 slurmm1.0-15647001.out filter=lfs diff=lfs merge=lfs -text
 wandb_256/offline-run-20260301_015115-wiagjqyt/run-wiagjqyt.wandb filter=lfs diff=lfs merge=lfs -text
+wandb_128/offline-run-20260227_135547-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text
+wandb_128/offline-run-20260227_141732-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text
+wandb_128/offline-run-20260301_014655-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text

wandb_128/debug.log

@@ -0,0 +1,25 @@
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Configure stats pid to 4004793
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-01 01:46:55,620 WARNING MainThread:4004793 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260301_014655-lxvmjzq8/logs/debug.log
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260301_014655-lxvmjzq8/logs/debug-internal.log
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():608] calling init triggers
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 128, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():658] starting backend
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():662] setting up manager
+2026-03-01 01:46:56,062 INFO    MainThread:4004793 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-01 01:46:56,070 INFO    MainThread:4004793 [wandb_init.py:init():670] backend started and connected
+2026-03-01 01:46:56,812 INFO    MainThread:4004793 [wandb_init.py:init():768] updated telemetry
+2026-03-01 01:46:56,868 INFO    MainThread:4004793 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-01 01:46:56,877 INFO    MainThread:4004793 [wandb_init.py:init():852] starting run threads in backend
+2026-03-01 01:47:03,232 INFO    MainThread:4004793 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-01 01:47:03,232 INFO    MainThread:4004793 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-01 01:47:03,233 INFO    MainThread:4004793 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-01 01:47:03,233 INFO    MainThread:4004793 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-01 01:47:03,244 INFO    MainThread:4004793 [wandb_init.py:init():895] run started, returning control to user process

wandb_128/offline-run-20260227_130446-lxvmjzq8/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

wandb_128/offline-run-20260227_130446-lxvmjzq8/files/wandb-metadata.json

@@ -0,0 +1,1361 @@
+{
+    "os": "Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-02-27T12:04:46.640569",
+    "startedAt": "2026-02-27T12:04:46.285930",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_a_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt",
+        "dataset=video_minecraft_latent",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=700",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_a_memory_pretrain",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.use_precomputed_features=true",
+        "algorithm.context_frames=1",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=8",
+        "experiment.training.batch_size=64",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=700000",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node020",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 128,
+    "cpu_count_logical": 256,
+    "cpu_freq": {
+        "current": 2440.1533984375005,
+        "min": 1500.0,
+        "max": 2250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3380.836,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.286,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.06,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.554,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.535,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.695,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.677,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.853,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.494,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.748,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.632,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.535,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.626,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.484,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.494,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.038,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.444,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.89,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3392.093,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.439,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.443,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.401,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.58,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.459,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.238,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.572,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.965,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.185,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.956,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.13,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.513,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.297,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.601,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.412,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.741,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.294,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.686,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.043,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3381.512,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3384.025,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 29.166847229003906,
+            "used": 15.824077606201172
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 2,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 1007.6652145385742
+    }
+}

wandb_128/offline-run-20260227_130446-lxvmjzq8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 11}}

wandb_128/offline-run-20260227_130446-lxvmjzq8/logs/debug-internal.log

@@ -0,0 +1,136 @@
+2026-02-27 13:04:46,338 INFO    StreamThr :1518997 [internal.py:wandb_internal():85] W&B internal server running at pid: 1518997, started at: 2026-02-27 13:04:46.335438
+2026-02-27 13:04:46,340 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: status
+2026-02-27 13:04:46,365 INFO    WriterThread:1518997 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_130446-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:04:46,476 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: run_start
+2026-02-27 13:04:46,515 DEBUG   HandlerThread:1518997 [system_info.py:__init__():26] System info init
+2026-02-27 13:04:46,515 DEBUG   HandlerThread:1518997 [system_info.py:__init__():41] System info init done
+2026-02-27 13:04:46,516 INFO    HandlerThread:1518997 [system_monitor.py:start():194] Starting system monitor
+2026-02-27 13:04:46,516 INFO    SystemMonitor:1518997 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-02-27 13:04:46,516 INFO    HandlerThread:1518997 [system_monitor.py:probe():214] Collecting system info
+2026-02-27 13:04:46,517 INFO    SystemMonitor:1518997 [interfaces.py:start():188] Started cpu monitoring
+2026-02-27 13:04:46,517 INFO    SystemMonitor:1518997 [interfaces.py:start():188] Started disk monitoring
+2026-02-27 13:04:46,518 INFO    SystemMonitor:1518997 [interfaces.py:start():188] Started gpu monitoring
+2026-02-27 13:04:46,518 INFO    SystemMonitor:1518997 [interfaces.py:start():188] Started memory monitoring
+2026-02-27 13:04:46,519 INFO    SystemMonitor:1518997 [interfaces.py:start():188] Started network monitoring
+2026-02-27 13:04:46,640 DEBUG   HandlerThread:1518997 [system_info.py:probe():152] Probing system
+2026-02-27 13:04:46,642 DEBUG   HandlerThread:1518997 [system_info.py:_probe_git():137] Probing git
+2026-02-27 13:04:46,725 DEBUG   HandlerThread:1518997 [system_info.py:_probe_git():145] Probing git done
+2026-02-27 13:04:46,725 DEBUG   HandlerThread:1518997 [system_info.py:probe():200] Probing system done
+2026-02-27 13:04:46,725 DEBUG   HandlerThread:1518997 [system_monitor.py:probe():223] {'os': 'Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28', 'python': '3.10.19', 'heartbeatAt': '2026-02-27T12:04:46.640569', 'startedAt': '2026-02-27T12:04:46.285930', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_a_mamba', 'algorithm=df_video_mamba3stage', '+customized_load=true', '+seperate_load=true', '+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', '+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'dataset=video_minecraft_latent', 'dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'dataset.n_frames=200', '+dataset.n_frames_valid=700', '+dataset.angle_range=110', '+dataset.pos_range=8', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', 'dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_a_memory_pretrain', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=false', 'algorithm.use_precomputed_features=true', 'algorithm.context_frames=1', '+algorithm.require_pose_prediction=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=8', 'experiment.training.batch_size=64', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=700000', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': 'f0d488aceb5425378cec21c5d35bd92f64346f51'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node020', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 128, 'cpu_count_logical': 256, 'cpu_freq': {'current': 2440.1533984375005, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3380.836, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.286, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.06, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.554, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.535, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.695, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.677, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.853, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.494, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.748, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.632, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.535, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.626, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.484, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.494, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.038, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.444, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.89, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3392.093, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.439, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3382.443, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3387.401, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.58, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.459, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.238, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.572, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.965, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.185, 'min': 1500.0, 'max': 2250.0}, {'current': 3385.956, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.13, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.513, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.297, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.601, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.412, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.741, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.294, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.686, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.043, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3381.512, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3384.025, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 29.166847229003906, 'used': 15.824077606201172}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 1007.6652145385742}}
+2026-02-27 13:04:46,725 INFO    HandlerThread:1518997 [system_monitor.py:probe():224] Finished collecting system info
+2026-02-27 13:04:46,725 INFO    HandlerThread:1518997 [system_monitor.py:probe():227] Publishing system info
+2026-02-27 13:04:46,726 DEBUG   HandlerThread:1518997 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-02-27 13:04:53,157 DEBUG   HandlerThread:1518997 [system_info.py:_save_conda():224] Saving conda packages done
+2026-02-27 13:04:53,164 INFO    HandlerThread:1518997 [system_monitor.py:probe():229] Finished publishing system info
+2026-02-27 13:04:53,178 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:04:53,178 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: keepalive
+2026-02-27 13:04:53,178 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:04:54,467 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: python_packages
+2026-02-27 13:04:57,179 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:04:57,180 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:04:58,193 DEBUG   SenderThread:1518997 [sender.py:send():391] send: exit
+2026-02-27 13:04:58,193 INFO    SenderThread:1518997 [sender.py:send_exit():598] handling exit code: 1
+2026-02-27 13:04:58,194 INFO    SenderThread:1518997 [sender.py:send_exit():600] handling runtime: 11
+2026-02-27 13:04:58,197 INFO    SenderThread:1518997 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2026-02-27 13:04:58,197 INFO    SenderThread:1518997 [sender.py:send_exit():606] send defer
+2026-02-27 13:04:58,197 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,197 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 0
+2026-02-27 13:04:58,197 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,197 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 0
+2026-02-27 13:04:58,197 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 1
+2026-02-27 13:04:58,197 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,197 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 1
+2026-02-27 13:04:58,198 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,198 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 1
+2026-02-27 13:04:58,198 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 2
+2026-02-27 13:04:58,198 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,198 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 2
+2026-02-27 13:04:58,198 INFO    HandlerThread:1518997 [system_monitor.py:finish():203] Stopping system monitor
+2026-02-27 13:04:58,198 DEBUG   SystemMonitor:1518997 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-02-27 13:04:58,198 DEBUG   SystemMonitor:1518997 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2026-02-27 13:04:58,198 DEBUG   SystemMonitor:1518997 [system_monitor.py:_start():183] Publishing last batch of metrics
+2026-02-27 13:04:58,199 INFO    HandlerThread:1518997 [interfaces.py:finish():200] Joined cpu monitor
+2026-02-27 13:04:58,200 INFO    HandlerThread:1518997 [interfaces.py:finish():200] Joined disk monitor
+2026-02-27 13:04:58,983 INFO    HandlerThread:1518997 [interfaces.py:finish():200] Joined gpu monitor
+2026-02-27 13:04:58,983 INFO    HandlerThread:1518997 [interfaces.py:finish():200] Joined memory monitor
+2026-02-27 13:04:58,984 INFO    HandlerThread:1518997 [interfaces.py:finish():200] Joined network monitor
+2026-02-27 13:04:58,984 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,984 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 2
+2026-02-27 13:04:58,984 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 3
+2026-02-27 13:04:58,984 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,984 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 3
+2026-02-27 13:04:58,984 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,984 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 3
+2026-02-27 13:04:58,984 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 4
+2026-02-27 13:04:58,984 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,985 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 4
+2026-02-27 13:04:58,985 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 4
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 5
+2026-02-27 13:04:58,985 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,985 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 5
+2026-02-27 13:04:58,985 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 5
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 6
+2026-02-27 13:04:58,985 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,985 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 6
+2026-02-27 13:04:58,985 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 6
+2026-02-27 13:04:58,985 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 7
+2026-02-27 13:04:58,986 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:04:58,986 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,986 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 7
+2026-02-27 13:04:58,986 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:04:58,986 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 7
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 8
+2026-02-27 13:04:58,986 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,986 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 8
+2026-02-27 13:04:58,986 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 8
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 9
+2026-02-27 13:04:58,986 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,986 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 9
+2026-02-27 13:04:58,986 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 9
+2026-02-27 13:04:58,986 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 10
+2026-02-27 13:04:58,987 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,987 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 10
+2026-02-27 13:04:58,987 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 10
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 11
+2026-02-27 13:04:58,987 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,987 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 11
+2026-02-27 13:04:58,987 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 11
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 12
+2026-02-27 13:04:58,987 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,987 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 12
+2026-02-27 13:04:58,987 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 12
+2026-02-27 13:04:58,987 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 13
+2026-02-27 13:04:58,988 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,988 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 13
+2026-02-27 13:04:58,988 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,988 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 13
+2026-02-27 13:04:58,988 INFO    SenderThread:1518997 [sender.py:transition_state():626] send defer: 14
+2026-02-27 13:04:58,988 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:04:58,988 INFO    HandlerThread:1518997 [handler.py:handle_request_defer():184] handle defer: 14
+2026-02-27 13:04:58,988 DEBUG   SenderThread:1518997 [sender.py:send():391] send: final
+2026-02-27 13:04:58,988 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:04:58,988 INFO    SenderThread:1518997 [sender.py:send_request_defer():622] handle sender defer: 14
+2026-02-27 13:04:58,989 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 13:04:58,989 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 13:04:58,990 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: server_info
+2026-02-27 13:04:58,990 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: server_info
+2026-02-27 13:04:58,990 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 13:04:58,990 DEBUG   SenderThread:1518997 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 13:04:58,990 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: get_summary
+2026-02-27 13:04:58,991 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: sampled_history
+2026-02-27 13:04:58,991 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: internal_messages
+2026-02-27 13:04:58,991 INFO    MainThread:1518997 [wandb_run.py:_footer_history_summary_info():4088] rendering history
+2026-02-27 13:04:58,991 INFO    MainThread:1518997 [wandb_run.py:_footer_history_summary_info():4120] rendering summary
+2026-02-27 13:04:58,991 DEBUG   HandlerThread:1518997 [handler.py:handle_request():158] handle_request: shutdown
+2026-02-27 13:04:58,991 INFO    HandlerThread:1518997 [handler.py:finish():884] shutting down handler
+2026-02-27 13:04:59,990 INFO    WriterThread:1518997 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_130446-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:04:59,990 INFO    SenderThread:1518997 [sender.py:finish():1631] shutting down sender

wandb_128/offline-run-20260227_130446-lxvmjzq8/logs/debug.log

@@ -0,0 +1,26 @@
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Configure stats pid to 1513219
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 13:04:46,313 WARNING MainThread:1513219 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_130446-lxvmjzq8/logs/debug.log
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_130446-lxvmjzq8/logs/debug-internal.log
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:init():608] calling init triggers
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 64, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:init():658] starting backend
+2026-02-27 13:04:46,313 INFO    MainThread:1513219 [wandb_init.py:init():662] setting up manager
+2026-02-27 13:04:46,333 INFO    MainThread:1513219 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 13:04:46,339 INFO    MainThread:1513219 [wandb_init.py:init():670] backend started and connected
+2026-02-27 13:04:46,365 INFO    MainThread:1513219 [wandb_init.py:init():768] updated telemetry
+2026-02-27 13:04:46,417 INFO    MainThread:1513219 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 13:04:46,423 INFO    MainThread:1513219 [wandb_init.py:init():852] starting run threads in backend
+2026-02-27 13:04:54,466 INFO    MainThread:1513219 [wandb_run.py:_console_start():2465] atexit reg
+2026-02-27 13:04:54,467 INFO    MainThread:1513219 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-02-27 13:04:54,467 INFO    MainThread:1513219 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-02-27 13:04:54,467 INFO    MainThread:1513219 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-02-27 13:04:54,480 INFO    MainThread:1513219 [wandb_init.py:init():895] run started, returning control to user process
+2026-02-27 13:04:59,996 WARNING MsgRouterThr:1513219 [router.py:message_loop():77] message_loop has been closed

wandb_128/offline-run-20260227_134842-lxvmjzq8/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

wandb_128/offline-run-20260227_134842-lxvmjzq8/files/wandb-metadata.json

@@ -0,0 +1,1361 @@
+{
+    "os": "Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-02-27T12:48:43.432625",
+    "startedAt": "2026-02-27T12:48:42.835440",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_a_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt",
+        "dataset=video_minecraft_latent",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=700",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_a_memory_pretrain",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.use_precomputed_features=true",
+        "algorithm.context_frames=1",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=8",
+        "experiment.training.batch_size=64",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=700000",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node043",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 128,
+    "cpu_count_logical": 256,
+    "cpu_freq": {
+        "current": 2285.5670195312505,
+        "min": 1500.0,
+        "max": 2250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3391.832,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3389.842,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3390.122,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3392.725,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3392.782,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.36,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.645,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 29.166847229003906,
+            "used": 15.821720123291016
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 2,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 1007.6507377624512
+    }
+}

wandb_128/offline-run-20260227_134842-lxvmjzq8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 80}}

wandb_128/offline-run-20260227_134842-lxvmjzq8/logs/debug-internal.log

@@ -0,0 +1,164 @@
+2026-02-27 13:48:42,881 INFO    StreamThr :1845920 [internal.py:wandb_internal():85] W&B internal server running at pid: 1845920, started at: 2026-02-27 13:48:42.879498
+2026-02-27 13:48:42,882 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status
+2026-02-27 13:48:43,210 INFO    WriterThread:1845920 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_134842-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:48:43,304 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: run_start
+2026-02-27 13:48:43,324 DEBUG   HandlerThread:1845920 [system_info.py:__init__():26] System info init
+2026-02-27 13:48:43,324 DEBUG   HandlerThread:1845920 [system_info.py:__init__():41] System info init done
+2026-02-27 13:48:43,324 INFO    HandlerThread:1845920 [system_monitor.py:start():194] Starting system monitor
+2026-02-27 13:48:43,324 INFO    SystemMonitor:1845920 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-02-27 13:48:43,325 INFO    HandlerThread:1845920 [system_monitor.py:probe():214] Collecting system info
+2026-02-27 13:48:43,325 INFO    SystemMonitor:1845920 [interfaces.py:start():188] Started cpu monitoring
+2026-02-27 13:48:43,326 INFO    SystemMonitor:1845920 [interfaces.py:start():188] Started disk monitoring
+2026-02-27 13:48:43,326 INFO    SystemMonitor:1845920 [interfaces.py:start():188] Started gpu monitoring
+2026-02-27 13:48:43,326 INFO    SystemMonitor:1845920 [interfaces.py:start():188] Started memory monitoring
+2026-02-27 13:48:43,327 INFO    SystemMonitor:1845920 [interfaces.py:start():188] Started network monitoring
+2026-02-27 13:48:43,432 DEBUG   HandlerThread:1845920 [system_info.py:probe():152] Probing system
+2026-02-27 13:48:43,434 DEBUG   HandlerThread:1845920 [system_info.py:_probe_git():137] Probing git
+2026-02-27 13:48:43,479 DEBUG   HandlerThread:1845920 [system_info.py:_probe_git():145] Probing git done
+2026-02-27 13:48:43,479 DEBUG   HandlerThread:1845920 [system_info.py:probe():200] Probing system done
+2026-02-27 13:48:43,479 DEBUG   HandlerThread:1845920 [system_monitor.py:probe():223] {'os': 'Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28', 'python': '3.10.19', 'heartbeatAt': '2026-02-27T12:48:43.432625', 'startedAt': '2026-02-27T12:48:42.835440', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_a_mamba', 'algorithm=df_video_mamba3stage', '+customized_load=true', '+seperate_load=true', '+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', '+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'dataset=video_minecraft_latent', 'dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'dataset.n_frames=200', '+dataset.n_frames_valid=700', '+dataset.angle_range=110', '+dataset.pos_range=8', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', 'dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_a_memory_pretrain', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=false', 'algorithm.use_precomputed_features=true', 'algorithm.context_frames=1', '+algorithm.require_pose_prediction=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=8', 'experiment.training.batch_size=64', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=700000', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': 'f0d488aceb5425378cec21c5d35bd92f64346f51'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node043', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 128, 'cpu_count_logical': 256, 'cpu_freq': {'current': 2285.5670195312505, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3391.832, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3389.842, 'min': 1500.0, 'max': 2250.0}, {'current': 3390.122, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3392.725, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3392.782, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.36, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.645, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 29.166847229003906, 'used': 15.821720123291016}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 1007.6507377624512}}
+2026-02-27 13:48:43,480 INFO    HandlerThread:1845920 [system_monitor.py:probe():224] Finished collecting system info
+2026-02-27 13:48:43,480 INFO    HandlerThread:1845920 [system_monitor.py:probe():227] Publishing system info
+2026-02-27 13:48:43,480 DEBUG   HandlerThread:1845920 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-02-27 13:48:50,058 DEBUG   HandlerThread:1845920 [system_info.py:_save_conda():224] Saving conda packages done
+2026-02-27 13:48:50,064 INFO    HandlerThread:1845920 [system_monitor.py:probe():229] Finished publishing system info
+2026-02-27 13:48:50,089 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:48:50,089 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: keepalive
+2026-02-27 13:48:50,090 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:48:51,668 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: python_packages
+2026-02-27 13:48:53,090 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:48:53,090 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:48:58,091 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:48:58,092 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:03,092 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:03,093 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:08,093 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:08,093 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:13,094 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:13,094 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:18,095 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:18,095 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:23,096 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:23,096 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:28,097 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:28,097 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:33,098 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:33,098 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:38,099 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:38,099 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:43,100 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:43,100 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:43,327 DEBUG   SystemMonitor:1845920 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-02-27 13:49:48,101 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:48,101 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:53,102 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:53,102 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:49:58,103 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:49:58,103 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:50:03,104 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:50:03,104 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:50:03,595 DEBUG   SenderThread:1845920 [sender.py:send():391] send: exit
+2026-02-27 13:50:03,595 INFO    SenderThread:1845920 [sender.py:send_exit():598] handling exit code: 0
+2026-02-27 13:50:03,595 INFO    SenderThread:1845920 [sender.py:send_exit():600] handling runtime: 80
+2026-02-27 13:50:03,600 INFO    SenderThread:1845920 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2026-02-27 13:50:03,601 INFO    SenderThread:1845920 [sender.py:send_exit():606] send defer
+2026-02-27 13:50:03,601 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:03,601 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 0
+2026-02-27 13:50:03,601 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:03,601 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 0
+2026-02-27 13:50:03,601 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 1
+2026-02-27 13:50:03,601 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:03,601 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 1
+2026-02-27 13:50:03,601 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:03,601 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 1
+2026-02-27 13:50:03,601 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 2
+2026-02-27 13:50:03,601 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:03,601 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 2
+2026-02-27 13:50:03,601 INFO    HandlerThread:1845920 [system_monitor.py:finish():203] Stopping system monitor
+2026-02-27 13:50:03,601 DEBUG   SystemMonitor:1845920 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2026-02-27 13:50:03,602 DEBUG   SystemMonitor:1845920 [system_monitor.py:_start():183] Publishing last batch of metrics
+2026-02-27 13:50:03,605 INFO    HandlerThread:1845920 [interfaces.py:finish():200] Joined cpu monitor
+2026-02-27 13:50:03,605 INFO    HandlerThread:1845920 [interfaces.py:finish():200] Joined disk monitor
+2026-02-27 13:50:04,450 INFO    HandlerThread:1845920 [interfaces.py:finish():200] Joined gpu monitor
+2026-02-27 13:50:04,450 INFO    HandlerThread:1845920 [interfaces.py:finish():200] Joined memory monitor
+2026-02-27 13:50:04,450 INFO    HandlerThread:1845920 [interfaces.py:finish():200] Joined network monitor
+2026-02-27 13:50:04,451 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,451 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 2
+2026-02-27 13:50:04,451 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 3
+2026-02-27 13:50:04,451 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,451 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 3
+2026-02-27 13:50:04,452 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,452 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 3
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 4
+2026-02-27 13:50:04,453 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,453 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 4
+2026-02-27 13:50:04,453 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 4
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 5
+2026-02-27 13:50:04,453 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,453 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 5
+2026-02-27 13:50:04,453 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 5
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 6
+2026-02-27 13:50:04,453 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,453 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 6
+2026-02-27 13:50:04,453 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 6
+2026-02-27 13:50:04,453 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 7
+2026-02-27 13:50:04,453 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:50:04,453 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,453 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 7
+2026-02-27 13:50:04,454 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:50:04,454 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 7
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 8
+2026-02-27 13:50:04,454 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,454 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 8
+2026-02-27 13:50:04,454 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 8
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 9
+2026-02-27 13:50:04,454 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,454 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 9
+2026-02-27 13:50:04,454 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 9
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 10
+2026-02-27 13:50:04,454 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,454 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 10
+2026-02-27 13:50:04,454 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 10
+2026-02-27 13:50:04,454 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 11
+2026-02-27 13:50:04,454 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,454 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 11
+2026-02-27 13:50:04,455 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 11
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 12
+2026-02-27 13:50:04,455 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,455 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 12
+2026-02-27 13:50:04,455 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 12
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 13
+2026-02-27 13:50:04,455 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,455 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 13
+2026-02-27 13:50:04,455 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 13
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:transition_state():626] send defer: 14
+2026-02-27 13:50:04,455 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 13:50:04,455 INFO    HandlerThread:1845920 [handler.py:handle_request_defer():184] handle defer: 14
+2026-02-27 13:50:04,455 DEBUG   SenderThread:1845920 [sender.py:send():391] send: final
+2026-02-27 13:50:04,455 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: defer
+2026-02-27 13:50:04,455 INFO    SenderThread:1845920 [sender.py:send_request_defer():622] handle sender defer: 14
+2026-02-27 13:50:04,456 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 13:50:04,456 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 13:50:04,456 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: server_info
+2026-02-27 13:50:04,457 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: server_info
+2026-02-27 13:50:04,457 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 13:50:04,457 DEBUG   SenderThread:1845920 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 13:50:04,457 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: get_summary
+2026-02-27 13:50:04,457 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: sampled_history
+2026-02-27 13:50:04,458 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: internal_messages
+2026-02-27 13:50:04,458 INFO    MainThread:1845920 [wandb_run.py:_footer_history_summary_info():4088] rendering history
+2026-02-27 13:50:04,458 INFO    MainThread:1845920 [wandb_run.py:_footer_history_summary_info():4120] rendering summary
+2026-02-27 13:50:04,459 DEBUG   HandlerThread:1845920 [handler.py:handle_request():158] handle_request: shutdown
+2026-02-27 13:50:04,459 INFO    HandlerThread:1845920 [handler.py:finish():884] shutting down handler
+2026-02-27 13:50:05,457 INFO    WriterThread:1845920 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_134842-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:50:05,457 INFO    SenderThread:1845920 [sender.py:finish():1631] shutting down sender

wandb_128/offline-run-20260227_134842-lxvmjzq8/logs/debug.log

@@ -0,0 +1,26 @@
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Configure stats pid to 1838592
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 13:48:42,865 WARNING MainThread:1838592 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_134842-lxvmjzq8/logs/debug.log
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_134842-lxvmjzq8/logs/debug-internal.log
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:init():608] calling init triggers
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 64, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:init():658] starting backend
+2026-02-27 13:48:42,865 INFO    MainThread:1838592 [wandb_init.py:init():662] setting up manager
+2026-02-27 13:48:42,878 INFO    MainThread:1838592 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 13:48:42,883 INFO    MainThread:1838592 [wandb_init.py:init():670] backend started and connected
+2026-02-27 13:48:43,210 INFO    MainThread:1838592 [wandb_init.py:init():768] updated telemetry
+2026-02-27 13:48:43,248 INFO    MainThread:1838592 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 13:48:43,253 INFO    MainThread:1838592 [wandb_init.py:init():852] starting run threads in backend
+2026-02-27 13:48:51,668 INFO    MainThread:1838592 [wandb_run.py:_console_start():2465] atexit reg
+2026-02-27 13:48:51,668 INFO    MainThread:1838592 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-02-27 13:48:51,668 INFO    MainThread:1838592 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-02-27 13:48:51,668 INFO    MainThread:1838592 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-02-27 13:48:51,680 INFO    MainThread:1838592 [wandb_init.py:init():895] run started, returning control to user process
+2026-02-27 13:50:05,461 WARNING MsgRouterThr:1838592 [router.py:message_loop():77] message_loop has been closed

wandb_128/offline-run-20260227_135547-lxvmjzq8/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

wandb_128/offline-run-20260227_135547-lxvmjzq8/files/wandb-metadata.json

@@ -0,0 +1,1361 @@
+{
+    "os": "Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-02-27T12:55:47.787240",
+    "startedAt": "2026-02-27T12:55:47.116648",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_a_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt",
+        "dataset=video_minecraft_latent",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=700",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_a_memory_pretrain",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.use_precomputed_features=true",
+        "algorithm.context_frames=1",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=8",
+        "experiment.training.batch_size=64",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=700000",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node043",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 128,
+    "cpu_count_logical": 256,
+    "cpu_freq": {
+        "current": 2285.5741328125005,
+        "min": 1500.0,
+        "max": 2250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.803,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3381.536,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3386.667,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3393.034,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3258.181,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3388.137,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3388.39,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.602,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3390.264,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.737,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 29.166847229003906,
+            "used": 15.821739196777344
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 2,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 1007.6507377624512
+    }
+}

wandb_128/offline-run-20260227_135547-lxvmjzq8/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 1078}}

wandb_128/offline-run-20260227_135547-lxvmjzq8/logs/debug-internal.log

@@ -0,0 +1,574 @@
+2026-02-27 13:55:47,166 INFO    StreamThr :1867587 [internal.py:wandb_internal():85] W&B internal server running at pid: 1867587, started at: 2026-02-27 13:55:47.163379
+2026-02-27 13:55:47,168 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status
+2026-02-27 13:55:47,520 INFO    WriterThread:1867587 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_135547-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:55:47,619 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: run_start
+2026-02-27 13:55:47,644 DEBUG   HandlerThread:1867587 [system_info.py:__init__():26] System info init
+2026-02-27 13:55:47,644 DEBUG   HandlerThread:1867587 [system_info.py:__init__():41] System info init done
+2026-02-27 13:55:47,644 INFO    HandlerThread:1867587 [system_monitor.py:start():194] Starting system monitor
+2026-02-27 13:55:47,644 INFO    SystemMonitor:1867587 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-02-27 13:55:47,644 INFO    HandlerThread:1867587 [system_monitor.py:probe():214] Collecting system info
+2026-02-27 13:55:47,644 INFO    SystemMonitor:1867587 [interfaces.py:start():188] Started cpu monitoring
+2026-02-27 13:55:47,645 INFO    SystemMonitor:1867587 [interfaces.py:start():188] Started disk monitoring
+2026-02-27 13:55:47,646 INFO    SystemMonitor:1867587 [interfaces.py:start():188] Started gpu monitoring
+2026-02-27 13:55:47,646 INFO    SystemMonitor:1867587 [interfaces.py:start():188] Started memory monitoring
+2026-02-27 13:55:47,646 INFO    SystemMonitor:1867587 [interfaces.py:start():188] Started network monitoring
+2026-02-27 13:55:47,787 DEBUG   HandlerThread:1867587 [system_info.py:probe():152] Probing system
+2026-02-27 13:55:47,789 DEBUG   HandlerThread:1867587 [system_info.py:_probe_git():137] Probing git
+2026-02-27 13:55:47,835 DEBUG   HandlerThread:1867587 [system_info.py:_probe_git():145] Probing git done
+2026-02-27 13:55:47,835 DEBUG   HandlerThread:1867587 [system_info.py:probe():200] Probing system done
+2026-02-27 13:55:47,835 DEBUG   HandlerThread:1867587 [system_monitor.py:probe():223] {'os': 'Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28', 'python': '3.10.19', 'heartbeatAt': '2026-02-27T12:55:47.787240', 'startedAt': '2026-02-27T12:55:47.116648', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_a_mamba', 'algorithm=df_video_mamba3stage', '+customized_load=true', '+seperate_load=true', '+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', '+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'dataset=video_minecraft_latent', 'dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'dataset.n_frames=200', '+dataset.n_frames_valid=700', '+dataset.angle_range=110', '+dataset.pos_range=8', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', 'dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_a_memory_pretrain', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=false', 'algorithm.use_precomputed_features=true', 'algorithm.context_frames=1', '+algorithm.require_pose_prediction=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=8', 'experiment.training.batch_size=64', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=700000', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': 'f0d488aceb5425378cec21c5d35bd92f64346f51'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node043', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 128, 'cpu_count_logical': 256, 'cpu_freq': {'current': 2285.5741328125005, 'min': 1500.0, 'max': 2250.0}, 'cpu_freq_per_core': [{'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3387.803, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3381.536, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3386.667, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3393.034, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3258.181, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3388.137, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3388.39, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3383.602, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3390.264, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 3387.737, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}, {'current': 2250.0, 'min': 1500.0, 'max': 2250.0}], 'disk': {'/': {'total': 29.166847229003906, 'used': 15.821739196777344}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 1007.6507377624512}}
+2026-02-27 13:55:47,835 INFO    HandlerThread:1867587 [system_monitor.py:probe():224] Finished collecting system info
+2026-02-27 13:55:47,835 INFO    HandlerThread:1867587 [system_monitor.py:probe():227] Publishing system info
+2026-02-27 13:55:47,835 DEBUG   HandlerThread:1867587 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-02-27 13:55:53,162 DEBUG   HandlerThread:1867587 [system_info.py:_save_conda():224] Saving conda packages done
+2026-02-27 13:55:53,169 INFO    HandlerThread:1867587 [system_monitor.py:probe():229] Finished publishing system info
+2026-02-27 13:55:53,180 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:55:53,180 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: keepalive
+2026-02-27 13:55:53,181 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:55:54,355 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: python_packages
+2026-02-27 13:55:57,181 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:55:57,181 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:02,182 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:02,182 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:07,183 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:07,183 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:12,184 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:12,184 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:17,185 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:17,186 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:22,187 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:22,187 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:27,188 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:27,188 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:32,189 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:32,189 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:37,190 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:37,190 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:42,191 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:42,191 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:47,192 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:47,192 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:47,646 DEBUG   SystemMonitor:1867587 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-02-27 13:56:52,193 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:52,194 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:56:57,195 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:56:57,195 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:02,196 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:02,196 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:07,197 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:07,197 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:12,198 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:12,198 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:17,199 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:17,199 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:22,200 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:22,200 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:27,201 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:27,201 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:32,202 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:32,202 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:37,203 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:37,203 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:42,204 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:42,204 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:47,205 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:47,205 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:52,207 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:52,207 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:57:57,208 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:57:57,208 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:02,209 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:02,209 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:07,210 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:07,210 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:12,211 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:12,212 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:17,212 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:17,213 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:22,214 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:22,214 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:27,215 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:27,215 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:32,216 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:32,216 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:37,217 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:37,217 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:37,927 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 13:58:40,765 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 13:58:42,218 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:42,219 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:47,220 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:47,220 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:52,221 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:52,221 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:58:57,222 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:58:57,222 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:02,223 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:02,223 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:07,224 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:07,225 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:12,225 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:12,226 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:17,227 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:17,227 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:22,228 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:22,228 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:27,229 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:27,229 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:32,230 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:32,230 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:37,231 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:37,231 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:42,232 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:42,232 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:47,233 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:47,233 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:52,234 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:52,234 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 13:59:57,235 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 13:59:57,235 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:02,236 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:02,236 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:07,237 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:07,237 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:12,238 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:12,238 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:17,239 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:17,239 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:22,240 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:22,240 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:27,242 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:27,242 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:32,244 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:32,244 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:37,245 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:37,245 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:42,246 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:42,247 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:47,248 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:47,248 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:52,249 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:52,249 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:00:57,250 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:00:57,250 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:02,251 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:02,252 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:07,253 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:07,253 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:12,254 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:12,254 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:15,340 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:01:17,255 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:17,255 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:18,170 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:01:22,256 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:22,256 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:27,257 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:27,257 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:32,258 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:32,258 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:37,259 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:37,259 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:42,260 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:42,260 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:47,261 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:47,261 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:52,262 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:52,262 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:01:57,263 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:01:57,263 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:02,264 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:02,264 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:07,265 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:07,265 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:12,266 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:12,266 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:17,267 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:17,267 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:22,268 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:22,268 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:27,269 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:27,269 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:32,270 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:32,270 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:37,271 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:37,271 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:42,272 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:42,272 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:47,273 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:47,273 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:52,274 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:52,274 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:02:57,275 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:02:57,275 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:02,276 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:02,276 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:07,277 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:07,277 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:12,278 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:12,278 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:17,279 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:17,279 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:22,280 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:22,280 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:27,281 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:27,282 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:32,282 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:32,283 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:37,284 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:37,284 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:42,285 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:42,285 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:47,286 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:47,286 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:52,287 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:52,287 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:03:52,748 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:03:56,235 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:03:57,288 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:03:57,288 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:02,289 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:02,290 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:07,291 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:07,291 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:12,292 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:12,292 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:17,293 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:17,293 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:22,294 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:22,295 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:27,295 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:27,296 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:32,297 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:32,297 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:37,298 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:37,298 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:42,299 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:42,299 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:47,300 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:47,300 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:52,301 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:52,301 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:04:57,302 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:04:57,302 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:02,303 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:02,303 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:07,305 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:07,306 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:12,306 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:12,307 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:17,308 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:17,308 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:22,309 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:22,309 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:27,310 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:27,310 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:32,311 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:32,311 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:37,312 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:37,312 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:42,313 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:42,314 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:47,314 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:47,315 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:52,315 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:52,316 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:05:57,317 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:05:57,317 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:02,318 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:02,318 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:07,319 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:07,319 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:12,320 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:12,320 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:17,321 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:17,321 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:22,322 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:22,322 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:27,323 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:27,323 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:29,417 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:06:32,324 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:32,324 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:32,865 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:06:37,325 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:37,325 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:42,326 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:42,326 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:47,327 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:47,327 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:52,328 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:52,328 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:06:57,329 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:06:57,329 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:02,330 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:02,331 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:07,331 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:07,332 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:12,333 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:12,333 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:17,334 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:17,334 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:22,335 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:22,335 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:27,336 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:27,336 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:32,337 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:32,337 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:37,338 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:37,338 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:42,339 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:42,339 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:47,340 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:47,340 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:52,341 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:52,341 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:07:57,342 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:07:57,342 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:02,343 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:02,343 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:07,344 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:07,344 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:12,345 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:12,345 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:17,346 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:17,346 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:22,347 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:22,348 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:27,348 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:27,349 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:32,350 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:32,350 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:37,351 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:37,351 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:42,352 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:42,352 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:47,352 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:47,353 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:52,354 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:52,354 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:08:57,354 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:08:57,355 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:02,355 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:02,356 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:06,091 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:09:07,356 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:07,356 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:08,934 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:09:12,357 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:12,357 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:17,358 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:17,358 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:22,359 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:22,359 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:27,360 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:27,360 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:32,361 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:32,361 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:37,362 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:37,362 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:42,363 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:42,363 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:47,364 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:47,364 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:52,365 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:52,365 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:09:57,366 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:09:57,366 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:02,367 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:02,367 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:07,368 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:07,368 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:12,368 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:12,369 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:17,369 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:17,370 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:22,370 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:22,371 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:27,371 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:27,372 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:32,372 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:32,373 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:37,373 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:37,373 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:42,374 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:42,374 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:47,375 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:47,375 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:52,376 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:52,376 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:10:57,377 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:10:57,377 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:02,378 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:02,378 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:07,379 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:07,379 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:12,380 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:12,380 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:17,380 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:17,381 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:22,381 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:22,382 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:27,382 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:27,383 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:32,383 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:32,383 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:37,384 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:37,384 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:42,385 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:42,385 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:42,743 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:11:45,551 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: partial_history
+2026-02-27 14:11:47,386 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:47,386 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:52,387 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:52,387 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:11:57,388 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:11:57,388 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:02,389 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:02,389 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:07,390 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:07,390 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:12,391 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:12,391 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:17,392 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:17,392 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:22,392 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:22,393 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:27,393 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:27,394 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:32,394 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:32,394 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:37,395 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:37,395 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:42,396 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:42,396 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:47,397 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:47,397 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:52,398 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:52,398 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:12:57,399 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:12:57,399 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:02,400 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:02,400 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:07,401 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:07,401 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:12,402 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:12,402 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:17,402 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:17,403 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:22,403 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:22,404 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:27,404 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:27,405 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:32,405 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:32,406 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:37,406 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:37,406 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:42,407 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:42,407 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:45,714 DEBUG   SenderThread:1867587 [sender.py:send():391] send: exit
+2026-02-27 14:13:45,715 INFO    SenderThread:1867587 [sender.py:send_exit():598] handling exit code: 0
+2026-02-27 14:13:45,715 INFO    SenderThread:1867587 [sender.py:send_exit():600] handling runtime: 1078
+2026-02-27 14:13:45,722 INFO    SenderThread:1867587 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2026-02-27 14:13:45,722 INFO    SenderThread:1867587 [sender.py:send_exit():606] send defer
+2026-02-27 14:13:45,722 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:45,722 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 0
+2026-02-27 14:13:45,722 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:45,722 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 0
+2026-02-27 14:13:45,722 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 1
+2026-02-27 14:13:45,722 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:45,723 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 1
+2026-02-27 14:13:45,723 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:45,723 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 1
+2026-02-27 14:13:45,723 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 2
+2026-02-27 14:13:45,723 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:45,723 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 2
+2026-02-27 14:13:45,723 INFO    HandlerThread:1867587 [system_monitor.py:finish():203] Stopping system monitor
+2026-02-27 14:13:45,723 DEBUG   SystemMonitor:1867587 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2026-02-27 14:13:45,723 DEBUG   SystemMonitor:1867587 [system_monitor.py:_start():183] Publishing last batch of metrics
+2026-02-27 14:13:45,727 INFO    HandlerThread:1867587 [interfaces.py:finish():200] Joined cpu monitor
+2026-02-27 14:13:45,727 INFO    HandlerThread:1867587 [interfaces.py:finish():200] Joined disk monitor
+2026-02-27 14:13:46,531 INFO    HandlerThread:1867587 [interfaces.py:finish():200] Joined gpu monitor
+2026-02-27 14:13:46,532 INFO    HandlerThread:1867587 [interfaces.py:finish():200] Joined memory monitor
+2026-02-27 14:13:46,532 INFO    HandlerThread:1867587 [interfaces.py:finish():200] Joined network monitor
+2026-02-27 14:13:46,532 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,532 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 2
+2026-02-27 14:13:46,532 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 3
+2026-02-27 14:13:46,532 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,532 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 3
+2026-02-27 14:13:46,533 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 3
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 4
+2026-02-27 14:13:46,533 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,533 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 4
+2026-02-27 14:13:46,533 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 4
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 5
+2026-02-27 14:13:46,533 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,533 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 5
+2026-02-27 14:13:46,533 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 5
+2026-02-27 14:13:46,533 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 6
+2026-02-27 14:13:46,533 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,533 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 6
+2026-02-27 14:13:46,533 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 6
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 7
+2026-02-27 14:13:46,534 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: status_report
+2026-02-27 14:13:46,534 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,534 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 7
+2026-02-27 14:13:46,534 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: status_report
+2026-02-27 14:13:46,534 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 7
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 8
+2026-02-27 14:13:46,534 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,534 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 8
+2026-02-27 14:13:46,534 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 8
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 9
+2026-02-27 14:13:46,534 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,534 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 9
+2026-02-27 14:13:46,534 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 9
+2026-02-27 14:13:46,534 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 10
+2026-02-27 14:13:46,534 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,534 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 10
+2026-02-27 14:13:46,535 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 10
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 11
+2026-02-27 14:13:46,535 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,535 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 11
+2026-02-27 14:13:46,535 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 11
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 12
+2026-02-27 14:13:46,535 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,535 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 12
+2026-02-27 14:13:46,535 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 12
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 13
+2026-02-27 14:13:46,535 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,535 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 13
+2026-02-27 14:13:46,535 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 13
+2026-02-27 14:13:46,535 INFO    SenderThread:1867587 [sender.py:transition_state():626] send defer: 14
+2026-02-27 14:13:46,535 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: defer
+2026-02-27 14:13:46,535 INFO    HandlerThread:1867587 [handler.py:handle_request_defer():184] handle defer: 14
+2026-02-27 14:13:46,536 DEBUG   SenderThread:1867587 [sender.py:send():391] send: final
+2026-02-27 14:13:46,536 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: defer
+2026-02-27 14:13:46,536 INFO    SenderThread:1867587 [sender.py:send_request_defer():622] handle sender defer: 14
+2026-02-27 14:13:46,536 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 14:13:46,536 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 14:13:46,537 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: server_info
+2026-02-27 14:13:46,537 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: server_info
+2026-02-27 14:13:46,537 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: poll_exit
+2026-02-27 14:13:46,537 DEBUG   SenderThread:1867587 [sender.py:send_request():418] send_request: poll_exit
+2026-02-27 14:13:46,537 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: get_summary
+2026-02-27 14:13:46,538 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: sampled_history
+2026-02-27 14:13:46,540 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: internal_messages
+2026-02-27 14:13:46,540 INFO    MainThread:1867587 [wandb_run.py:_footer_history_summary_info():4088] rendering history
+2026-02-27 14:13:46,540 INFO    MainThread:1867587 [wandb_run.py:_footer_history_summary_info():4120] rendering summary
+2026-02-27 14:13:46,541 DEBUG   HandlerThread:1867587 [handler.py:handle_request():158] handle_request: shutdown
+2026-02-27 14:13:46,541 INFO    HandlerThread:1867587 [handler.py:finish():884] shutting down handler
+2026-02-27 14:13:47,537 INFO    WriterThread:1867587 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_135547-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 14:13:47,537 INFO    SenderThread:1867587 [sender.py:finish():1631] shutting down sender

wandb_128/offline-run-20260227_135547-lxvmjzq8/logs/debug.log

@@ -0,0 +1,26 @@
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Configure stats pid to 1861603
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 13:55:47,146 WARNING MainThread:1861603 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_135547-lxvmjzq8/logs/debug.log
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_135547-lxvmjzq8/logs/debug-internal.log
+2026-02-27 13:55:47,146 INFO    MainThread:1861603 [wandb_init.py:init():608] calling init triggers
+2026-02-27 13:55:47,147 INFO    MainThread:1861603 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 64, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 13:55:47,147 INFO    MainThread:1861603 [wandb_init.py:init():658] starting backend
+2026-02-27 13:55:47,147 INFO    MainThread:1861603 [wandb_init.py:init():662] setting up manager
+2026-02-27 13:55:47,162 INFO    MainThread:1861603 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 13:55:47,172 INFO    MainThread:1861603 [wandb_init.py:init():670] backend started and connected
+2026-02-27 13:55:47,519 INFO    MainThread:1861603 [wandb_init.py:init():768] updated telemetry
+2026-02-27 13:55:47,561 INFO    MainThread:1861603 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 13:55:47,567 INFO    MainThread:1861603 [wandb_init.py:init():852] starting run threads in backend
+2026-02-27 13:55:54,355 INFO    MainThread:1861603 [wandb_run.py:_console_start():2465] atexit reg
+2026-02-27 13:55:54,355 INFO    MainThread:1861603 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-02-27 13:55:54,355 INFO    MainThread:1861603 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-02-27 13:55:54,355 INFO    MainThread:1861603 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-02-27 13:55:54,361 INFO    MainThread:1861603 [wandb_init.py:init():895] run started, returning control to user process
+2026-02-27 14:13:47,544 WARNING MsgRouterThr:1861603 [router.py:message_loop():77] message_loop has been closed

wandb_128/offline-run-20260227_135547-lxvmjzq8/run-lxvmjzq8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:482d56811ef60d6dbed540ea92d080298d92f0dfb0461c9bc0fabcaec49f2d1d
+size 541322

wandb_128/offline-run-20260227_141732-lxvmjzq8/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

wandb_128/offline-run-20260227_141732-lxvmjzq8/files/wandb-metadata.json

@@ -0,0 +1,1361 @@
+{
+    "os": "Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-02-27T13:17:33.386855",
+    "startedAt": "2026-02-27T13:17:32.755782",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_a_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt",
+        "dataset=video_minecraft_latent",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=700",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_a_memory_pretrain",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.use_precomputed_features=true",
+        "algorithm.context_frames=1",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=8",
+        "experiment.training.batch_size=128",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=700000",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node043",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 128,
+    "cpu_count_logical": 256,
+    "cpu_freq": {
+        "current": 2281.076578125,
+        "min": 1500.0,
+        "max": 2250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3382.953,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3389.431,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.824,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3385.714,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3388.259,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.702,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.131,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3393.697,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3393.573,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 29.166847229003906,
+            "used": 15.821796417236328
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-40GB",
+    "gpu_count": 2,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        },
+        {
+            "name": "NVIDIA A100-SXM4-40GB",
+            "memory_total": 42949672960
+        }
+    ],
+    "memory": {
+        "total": 1007.6507377624512
+    }
+}

wandb_128/offline-run-20260227_141732-lxvmjzq8/logs/debug.log

@@ -0,0 +1,25 @@
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Configure stats pid to 1937395
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-02-27 14:17:32,778 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 14:17:32,778 WARNING MainThread:1937395 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_141732-lxvmjzq8/logs/debug.log
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260227_141732-lxvmjzq8/logs/debug-internal.log
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:init():608] calling init triggers
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 128, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:init():658] starting backend
+2026-02-27 14:17:32,779 INFO    MainThread:1937395 [wandb_init.py:init():662] setting up manager
+2026-02-27 14:17:32,795 INFO    MainThread:1937395 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 14:17:32,802 INFO    MainThread:1937395 [wandb_init.py:init():670] backend started and connected
+2026-02-27 14:17:33,125 INFO    MainThread:1937395 [wandb_init.py:init():768] updated telemetry
+2026-02-27 14:17:33,168 INFO    MainThread:1937395 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 14:17:33,172 INFO    MainThread:1937395 [wandb_init.py:init():852] starting run threads in backend
+2026-02-27 14:17:39,731 INFO    MainThread:1937395 [wandb_run.py:_console_start():2465] atexit reg
+2026-02-27 14:17:39,732 INFO    MainThread:1937395 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-02-27 14:17:39,732 INFO    MainThread:1937395 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-02-27 14:17:39,732 INFO    MainThread:1937395 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-02-27 14:17:39,741 INFO    MainThread:1937395 [wandb_init.py:init():895] run started, returning control to user process

wandb_128/offline-run-20260227_141732-lxvmjzq8/run-lxvmjzq8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9683fc21ede21958918726a0fd092e1e08de14b6a4367f492c097a1ea5d95c49
+size 892663

wandb_128/offline-run-20260301_014655-lxvmjzq8/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

wandb_128/offline-run-20260301_014655-lxvmjzq8/files/wandb-metadata.json

@@ -0,0 +1,1385 @@
+{
+    "os": "Linux-4.18.0-553.94.1.el8_10.x86_64-x86_64-with-glibc2.28",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-03-01T00:46:57.077460",
+    "startedAt": "2026-03-01T00:46:55.595381",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_a_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=true",
+        "+diffusion_model_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt",
+        "+vae_path=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt",
+        "dataset=video_minecraft_latent",
+        "dataset.precomputed_feature_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=700",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_a_memory_pretrain",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.use_precomputed_features=true",
+        "algorithm.context_frames=1",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=8",
+        "experiment.training.batch_size=128",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=700000",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node063",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 128,
+    "cpu_count_logical": 256,
+    "cpu_freq": {
+        "current": 2289.95471484375,
+        "min": 1500.0,
+        "max": 2250.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3388.711,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.933,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.589,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3388.154,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3389.651,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3383.532,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3387.929,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 3381.246,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        },
+        {
+            "current": 2250.0,
+            "min": 1500.0,
+            "max": 2250.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 29.166847229003906,
+            "used": 15.824031829833984
+        }
+    },
+    "gpu": "NVIDIA A100-SXM4-80GB",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        },
+        {
+            "name": "NVIDIA A100-SXM4-80GB",
+            "memory_total": 85899345920
+        }
+    ],
+    "memory": {
+        "total": 2015.6356811523438
+    }
+}

wandb_128/offline-run-20260301_014655-lxvmjzq8/logs/debug.log

@@ -0,0 +1,25 @@
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Configure stats pid to 4004793
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-01 01:46:55,620 WARNING MainThread:4004793 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260301_014655-lxvmjzq8/logs/debug.log
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/offline-run-20260301_014655-lxvmjzq8/logs/debug-internal.log
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():608] calling init triggers
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 128, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():658] starting backend
+2026-03-01 01:46:55,620 INFO    MainThread:4004793 [wandb_init.py:init():662] setting up manager
+2026-03-01 01:46:56,062 INFO    MainThread:4004793 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-01 01:46:56,070 INFO    MainThread:4004793 [wandb_init.py:init():670] backend started and connected
+2026-03-01 01:46:56,812 INFO    MainThread:4004793 [wandb_init.py:init():768] updated telemetry
+2026-03-01 01:46:56,868 INFO    MainThread:4004793 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-01 01:46:56,877 INFO    MainThread:4004793 [wandb_init.py:init():852] starting run threads in backend
+2026-03-01 01:47:03,232 INFO    MainThread:4004793 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-01 01:47:03,232 INFO    MainThread:4004793 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-01 01:47:03,233 INFO    MainThread:4004793 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-01 01:47:03,233 INFO    MainThread:4004793 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-01 01:47:03,244 INFO    MainThread:4004793 [wandb_init.py:init():895] run started, returning control to user process

wandb_128/offline-run-20260301_014655-lxvmjzq8/run-lxvmjzq8.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58d6d2a235b953c1ef8fd60a6fd814e0126d68f9bfd8f91c51f487f08c1d3ae6
+size 68717297

wandb_128/run-20260227_130222-lxvmjzq8/files/config.yaml

@@ -0,0 +1,254 @@
+wandb_version: 1
+
+experiment:
+  desc: null
+  value:
+    debug: ${debug}
+    tasks:
+    - training
+    num_nodes: 1
+    training:
+      precision: 16-mixed
+      compile: false
+      lr: 2.0e-05
+      batch_size: 64
+      max_epochs: -1
+      max_steps: 700000
+      max_time: null
+      data:
+        num_workers: 4
+        shuffle: true
+      optim:
+        accumulate_grad_batches: 1
+        gradient_clip_val: 1.0
+      checkpointing:
+        every_n_train_steps: 2500
+        every_n_epochs: null
+        train_time_interval: null
+        enable_version_counter: false
+    validation:
+      precision: 16-mixed
+      compile: false
+      batch_size: 4
+      val_every_n_step: 2500
+      val_every_n_epoch: null
+      limit_batch: 1
+      inference_mode: false
+      data:
+        num_workers: 4
+        shuffle: false
+    test:
+      precision: 16-mixed
+      compile: false
+      batch_size: 1
+      limit_batch: 1
+      inference_mode: false
+      data:
+        num_workers: 4
+        shuffle: false
+    logging:
+      metrics: null
+    _name: exp_video
+dataset:
+  desc: null
+  value:
+    debug: ${debug}
+    metadata: data/${dataset.name}/metadata.json
+    data_mean: 0.5
+    data_std: 0.5
+    save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
+    n_frames: 200
+    context_length: 1
+    resolution: 128
+    observation_shape:
+    - 3
+    - ${dataset.resolution}
+    - ${dataset.resolution}
+    external_cond_dim: 0
+    validation_multiplier: 1
+    frame_skip: 1
+    action_cond_dim: 25
+    _name: video_minecraft_latent
+    precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
+    use_explicit_memory_frames: false
+    n_frames_valid: 700
+    angle_range: 110
+    pos_range: 8
+    customized_validation: true
+    add_timestamp_embedding: true
+algorithm:
+  desc: null
+  value:
+    debug: ${debug}
+    lr: ${experiment.training.lr}
+    x_shape: ${dataset.observation_shape}
+    frame_stack: 1
+    frame_skip: ${dataset.frame_skip}
+    data_mean: ${dataset.data_mean}
+    data_std: ${dataset.data_std}
+    external_cond_dim: 0
+    context_frames: 1
+    weight_decay: 0.002
+    warmup_steps: 1000
+    optimizer_beta:
+    - 0.9
+    - 0.99
+    uncertainty_scale: 1
+    guidance_scale: 0.0
+    chunk_size: 1
+    scheduling_matrix: autoregressive
+    noise_level: random_all
+    causal: true
+    diffusion:
+      objective: pred_v
+      beta_schedule: sigmoid
+      schedule_fn_kwargs: {}
+      clip_noise: 20.0
+      use_snr: false
+      use_cum_snr: false
+      use_fused_snr: true
+      snr_clip: 5.0
+      cum_snr_decay: 0.96
+      timesteps: 1000
+      sampling_timesteps: 20
+      ddim_sampling_eta: 0.0
+      stabilization_level: 15
+      architecture:
+        network_size: 64
+        attn_heads: 4
+        attn_dim_head: 64
+        dim_mults:
+        - 1
+        - 2
+        - 4
+        - 8
+        resolution: ${dataset.resolution}
+        attn_resolutions:
+        - 16
+        - 32
+        - 64
+        - 128
+        use_init_temporal_attn: true
+        use_linear_attn: true
+        time_emb_type: rotary
+    n_frames: ${dataset.n_frames}
+    metadata: ${dataset.metadata}
+    action_cond_dim: 25
+    use_plucker: true
+    memory_condition_length: 0
+    log_video: true
+    use_compressed_causal_memory: false
+    compressed_memory_dim: 256
+    compressed_memory_slots: 64
+    compressed_memory_topk: 4
+    use_mamba_memory_pipeline: true
+    training_stage: stage_a_memory_pretrain
+    stage_c_memory_aux_weight: 0.1
+    diff_window_size: 8
+    memory_gap_aux_weight: 0.1
+    memory_gap_aux_weight_init: 0.5
+    memory_gap_aux_weight_final: 0.1
+    memory_gap_aux_decay_fraction: 0.3
+    memory_gap_aux_decay_steps: 0
+    memory_gap_aux_exp_k: 5.0
+    use_precomputed_features: true
+    mamba_latent_channels: 16
+    mamba_model_dim: 256
+    mamba_depth: 4
+    mamba_cond_dim: 256
+    mamba_d_state: 16
+    mamba_d_conv: 4
+    mamba_expand: 2
+    allow_mamba_fallback: false
+    strict_causal_training: true
+    strict_causal_evaluation: true
+    use_oracle_pose_eval: false
+    enable_memory_noise_curriculum: true
+    curriculum_phase_boundaries:
+    - 0.2
+    - 0.7
+    curriculum_noise_ranges:
+    - - 600
+      - 1000
+    - - 200
+      - 900
+    - - 0
+      - 400
+    curriculum_horizons:
+    - 8
+    - 64
+    - 256
+    _name: df_video_mamba3stage
+    require_pose_prediction: false
+    use_memory_attention: false
+    relative_embedding: false
+    memory_retrieval_topk: 8
+debug:
+  desc: null
+  value: false
+wandb:
+  desc: null
+  value:
+    entity: turlin
+    project: worldmem
+    mode: online
+resume:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: null
+name:
+  desc: null
+  value: train_stage_a_mamba
+customized_load:
+  desc: null
+  value: true
+seperate_load:
+  desc: null
+  value: true
+diffusion_model_path:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt
+vae_path:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt
+output_dir:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.19
+    cli_version: 0.17.9
+    framework: huggingface
+    huggingface_version: 5.2.0
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1772193742
+    t:
+      1:
+      - 1
+      - 9
+      - 11
+      - 29
+      - 41
+      - 49
+      - 50
+      - 51
+      - 55
+      - 63
+      - 103
+      - 105
+      - 106
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.19
+      5: 0.17.9
+      6: 5.2.0
+      8:
+      - 2
+      - 5
+      13: linux-x86_64

wandb_128/run-20260227_130222-lxvmjzq8/logs/debug-internal.log

@@ -0,0 +1,50 @@
+2026-02-27 13:02:22,464 INFO    StreamThr :1510588 [internal.py:wandb_internal():85] W&B internal server running at pid: 1510588, started at: 2026-02-27 13:02:22.460261
+2026-02-27 13:02:22,465 DEBUG   HandlerThread:1510588 [handler.py:handle_request():158] handle_request: status
+2026-02-27 13:02:23,404 INFO    WriterThread:1510588 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_130222-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:02:23,407 DEBUG   SenderThread:1510588 [sender.py:send():391] send: header
+2026-02-27 13:02:23,489 DEBUG   SenderThread:1510588 [sender.py:send():391] send: run
+2026-02-27 13:02:23,507 INFO    SenderThread:1510588 [sender.py:_setup_resume():761] checking resume status for turlin/worldmem/lxvmjzq8
+2026-02-27 13:02:24,031 ERROR   SenderThread:1510588 [internal_api.py:execute():349] 403 response executing GraphQL.
+2026-02-27 13:02:24,031 ERROR   SenderThread:1510588 [internal_api.py:execute():350] {"errors":[{"message":"permission denied","path":["upsertBucket"],"extensions":{"code":"PERMISSION_ERROR"}}],"data":{"upsertBucket":null}}
+2026-02-27 13:02:24,032 ERROR   SenderThread:1510588 [sender.py:send_run():1021] It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 131, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 345, in execute
+    return self.client.execute(*args, **kwargs)  # type: ignore
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/gql_request.py", line 59, in execute
+    request.raise_for_status()
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status
+    raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://api.wandb.ai/graphql
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/sender.py", line 1019, in send_run
+    server_run = self._init_run(run, config_value_dict)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/sender.py", line 1091, in _init_run
+    server_run, inserted, server_messages = self._api.upsert_run(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/apis/normalize.py", line 73, in wrapper
+    raise err
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/apis/normalize.py", line 41, in wrapper
+    return func(*args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 2224, in upsert_run
+    response = self.gql(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 317, in gql
+    ret = self._retry_gql(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 147, in __call__
+    retry_timedelta_triggered = check_retry_fn(e)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/util.py", line 968, in check_retry_fn
+    return fallback_retry_fn(e)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/util.py", line 910, in no_retry_auth
+    raise CommError(
+wandb.errors.CommError: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:02:24,190 DEBUG   HandlerThread:1510588 [handler.py:handle_request():158] handle_request: shutdown
+2026-02-27 13:02:24,190 INFO    HandlerThread:1510588 [handler.py:finish():884] shutting down handler
+2026-02-27 13:02:24,489 INFO    WriterThread:1510588 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_130222-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:02:25,058 INFO    SenderThread:1510588 [sender.py:finish():1631] shutting down sender

wandb_128/run-20260227_130222-lxvmjzq8/logs/debug.log

@@ -0,0 +1,29 @@
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Configure stats pid to 1505055
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 13:02:22,434 WARNING MainThread:1505055 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_setup.py:_flush():77] Applying login settings: {}
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_130222-lxvmjzq8/logs/debug.log
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_130222-lxvmjzq8/logs/debug-internal.log
+2026-02-27 13:02:22,434 INFO    MainThread:1505055 [wandb_init.py:init():608] calling init triggers
+2026-02-27 13:02:22,435 INFO    MainThread:1505055 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 64, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 13:02:22,435 INFO    MainThread:1505055 [wandb_init.py:init():658] starting backend
+2026-02-27 13:02:22,435 INFO    MainThread:1505055 [wandb_init.py:init():662] setting up manager
+2026-02-27 13:02:22,459 INFO    MainThread:1505055 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 13:02:22,468 INFO    MainThread:1505055 [wandb_init.py:init():670] backend started and connected
+2026-02-27 13:02:23,404 INFO    MainThread:1505055 [wandb_init.py:init():768] updated telemetry
+2026-02-27 13:02:23,488 INFO    MainThread:1505055 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 13:02:24,058 ERROR   MainThread:1505055 [wandb_init.py:init():827] encountered error: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:02:24,058 ERROR   MainThread:1505055 [wandb_init.py:init():1239] error in wandb.init()
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1229, in init
+    return wi.init()
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 833, in init
+    raise error
+wandb.errors.CommError: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:02:25,191 WARNING MsgRouterThr:1505055 [router.py:message_loop():77] message_loop has been closed

wandb_128/run-20260227_134636-lxvmjzq8/files/config.yaml

@@ -0,0 +1,254 @@
+wandb_version: 1
+
+experiment:
+  desc: null
+  value:
+    debug: ${debug}
+    tasks:
+    - training
+    num_nodes: 1
+    training:
+      precision: 16-mixed
+      compile: false
+      lr: 2.0e-05
+      batch_size: 64
+      max_epochs: -1
+      max_steps: 700000
+      max_time: null
+      data:
+        num_workers: 4
+        shuffle: true
+      optim:
+        accumulate_grad_batches: 1
+        gradient_clip_val: 1.0
+      checkpointing:
+        every_n_train_steps: 2500
+        every_n_epochs: null
+        train_time_interval: null
+        enable_version_counter: false
+    validation:
+      precision: 16-mixed
+      compile: false
+      batch_size: 4
+      val_every_n_step: 2500
+      val_every_n_epoch: null
+      limit_batch: 1
+      inference_mode: false
+      data:
+        num_workers: 4
+        shuffle: false
+    test:
+      precision: 16-mixed
+      compile: false
+      batch_size: 1
+      limit_batch: 1
+      inference_mode: false
+      data:
+        num_workers: 4
+        shuffle: false
+    logging:
+      metrics: null
+    _name: exp_video
+dataset:
+  desc: null
+  value:
+    debug: ${debug}
+    metadata: data/${dataset.name}/metadata.json
+    data_mean: 0.5
+    data_std: 0.5
+    save_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft
+    n_frames: 200
+    context_length: 1
+    resolution: 128
+    observation_shape:
+    - 3
+    - ${dataset.resolution}
+    - ${dataset.resolution}
+    external_cond_dim: 0
+    validation_multiplier: 1
+    frame_skip: 1
+    action_cond_dim: 25
+    _name: video_minecraft_latent
+    precomputed_feature_dir: /proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features
+    use_explicit_memory_frames: false
+    n_frames_valid: 700
+    angle_range: 110
+    pos_range: 8
+    customized_validation: true
+    add_timestamp_embedding: true
+algorithm:
+  desc: null
+  value:
+    debug: ${debug}
+    lr: ${experiment.training.lr}
+    x_shape: ${dataset.observation_shape}
+    frame_stack: 1
+    frame_skip: ${dataset.frame_skip}
+    data_mean: ${dataset.data_mean}
+    data_std: ${dataset.data_std}
+    external_cond_dim: 0
+    context_frames: 1
+    weight_decay: 0.002
+    warmup_steps: 1000
+    optimizer_beta:
+    - 0.9
+    - 0.99
+    uncertainty_scale: 1
+    guidance_scale: 0.0
+    chunk_size: 1
+    scheduling_matrix: autoregressive
+    noise_level: random_all
+    causal: true
+    diffusion:
+      objective: pred_v
+      beta_schedule: sigmoid
+      schedule_fn_kwargs: {}
+      clip_noise: 20.0
+      use_snr: false
+      use_cum_snr: false
+      use_fused_snr: true
+      snr_clip: 5.0
+      cum_snr_decay: 0.96
+      timesteps: 1000
+      sampling_timesteps: 20
+      ddim_sampling_eta: 0.0
+      stabilization_level: 15
+      architecture:
+        network_size: 64
+        attn_heads: 4
+        attn_dim_head: 64
+        dim_mults:
+        - 1
+        - 2
+        - 4
+        - 8
+        resolution: ${dataset.resolution}
+        attn_resolutions:
+        - 16
+        - 32
+        - 64
+        - 128
+        use_init_temporal_attn: true
+        use_linear_attn: true
+        time_emb_type: rotary
+    n_frames: ${dataset.n_frames}
+    metadata: ${dataset.metadata}
+    action_cond_dim: 25
+    use_plucker: true
+    memory_condition_length: 0
+    log_video: true
+    use_compressed_causal_memory: false
+    compressed_memory_dim: 256
+    compressed_memory_slots: 64
+    compressed_memory_topk: 4
+    use_mamba_memory_pipeline: true
+    training_stage: stage_a_memory_pretrain
+    stage_c_memory_aux_weight: 0.1
+    diff_window_size: 8
+    memory_gap_aux_weight: 0.1
+    memory_gap_aux_weight_init: 0.5
+    memory_gap_aux_weight_final: 0.1
+    memory_gap_aux_decay_fraction: 0.3
+    memory_gap_aux_decay_steps: 0
+    memory_gap_aux_exp_k: 5.0
+    use_precomputed_features: true
+    mamba_latent_channels: 16
+    mamba_model_dim: 256
+    mamba_depth: 4
+    mamba_cond_dim: 256
+    mamba_d_state: 16
+    mamba_d_conv: 4
+    mamba_expand: 2
+    allow_mamba_fallback: false
+    strict_causal_training: true
+    strict_causal_evaluation: true
+    use_oracle_pose_eval: false
+    enable_memory_noise_curriculum: true
+    curriculum_phase_boundaries:
+    - 0.2
+    - 0.7
+    curriculum_noise_ranges:
+    - - 600
+      - 1000
+    - - 200
+      - 900
+    - - 0
+      - 400
+    curriculum_horizons:
+    - 8
+    - 64
+    - 256
+    _name: df_video_mamba3stage
+    require_pose_prediction: false
+    use_memory_attention: false
+    relative_embedding: false
+    memory_retrieval_topk: 8
+debug:
+  desc: null
+  value: false
+wandb:
+  desc: null
+  value:
+    entity: turlin
+    project: worldmem
+    mode: online
+resume:
+  desc: null
+  value: null
+load:
+  desc: null
+  value: null
+name:
+  desc: null
+  value: train_stage_a_mamba
+customized_load:
+  desc: null
+  value: true
+seperate_load:
+  desc: null
+  value: true
+diffusion_model_path:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt
+vae_path:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt
+output_dir:
+  desc: null
+  value: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/
+_wandb:
+  desc: null
+  value:
+    python_version: 3.10.19
+    cli_version: 0.17.9
+    framework: huggingface
+    huggingface_version: 5.2.0
+    is_jupyter_run: false
+    is_kaggle_kernel: true
+    start_time: 1772196396
+    t:
+      1:
+      - 1
+      - 9
+      - 11
+      - 29
+      - 41
+      - 49
+      - 50
+      - 51
+      - 55
+      - 63
+      - 103
+      - 105
+      - 106
+      3:
+      - 13
+      - 16
+      - 23
+      4: 3.10.19
+      5: 0.17.9
+      6: 5.2.0
+      8:
+      - 2
+      - 5
+      13: linux-x86_64

wandb_128/run-20260227_134636-lxvmjzq8/logs/debug-internal.log

@@ -0,0 +1,50 @@
+2026-02-27 13:46:36,588 INFO    StreamThr :1836908 [internal.py:wandb_internal():85] W&B internal server running at pid: 1836908, started at: 2026-02-27 13:46:36.586733
+2026-02-27 13:46:36,590 DEBUG   HandlerThread:1836908 [handler.py:handle_request():158] handle_request: status
+2026-02-27 13:46:36,612 INFO    WriterThread:1836908 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_134636-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:46:36,614 DEBUG   SenderThread:1836908 [sender.py:send():391] send: header
+2026-02-27 13:46:36,663 DEBUG   SenderThread:1836908 [sender.py:send():391] send: run
+2026-02-27 13:46:36,679 INFO    SenderThread:1836908 [sender.py:_setup_resume():761] checking resume status for turlin/worldmem/lxvmjzq8
+2026-02-27 13:46:37,220 ERROR   SenderThread:1836908 [internal_api.py:execute():349] 403 response executing GraphQL.
+2026-02-27 13:46:37,220 ERROR   SenderThread:1836908 [internal_api.py:execute():350] {"errors":[{"message":"permission denied","path":["upsertBucket"],"extensions":{"code":"PERMISSION_ERROR"}}],"data":{"upsertBucket":null}}
+2026-02-27 13:46:37,220 ERROR   SenderThread:1836908 [sender.py:send_run():1021] It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 131, in __call__
+    result = self._call_fn(*args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 345, in execute
+    return self.client.execute(*args, **kwargs)  # type: ignore
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 52, in execute
+    result = self._get_result(document, *args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/vendor/gql-0.2.0/wandb_gql/client.py", line 60, in _get_result
+    return self.transport.execute(document, *args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/gql_request.py", line 59, in execute
+    request.raise_for_status()
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/requests/models.py", line 1026, in raise_for_status
+    raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 403 Client Error: Forbidden for url: https://api.wandb.ai/graphql
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/sender.py", line 1019, in send_run
+    server_run = self._init_run(run, config_value_dict)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/sender.py", line 1091, in _init_run
+    server_run, inserted, server_messages = self._api.upsert_run(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/apis/normalize.py", line 73, in wrapper
+    raise err
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/apis/normalize.py", line 41, in wrapper
+    return func(*args, **kwargs)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 2224, in upsert_run
+    response = self.gql(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/internal/internal_api.py", line 317, in gql
+    ret = self._retry_gql(
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/lib/retry.py", line 147, in __call__
+    retry_timedelta_triggered = check_retry_fn(e)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/util.py", line 968, in check_retry_fn
+    return fallback_retry_fn(e)
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/util.py", line 910, in no_retry_auth
+    raise CommError(
+wandb.errors.CommError: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:46:37,381 DEBUG   HandlerThread:1836908 [handler.py:handle_request():158] handle_request: shutdown
+2026-02-27 13:46:37,381 INFO    HandlerThread:1836908 [handler.py:finish():884] shutting down handler
+2026-02-27 13:46:37,663 INFO    WriterThread:1836908 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_134636-lxvmjzq8/run-lxvmjzq8.wandb
+2026-02-27 13:46:38,245 INFO    SenderThread:1836908 [sender.py:finish():1631] shutting down sender

wandb_128/run-20260227_134636-lxvmjzq8/logs/debug.log

@@ -0,0 +1,29 @@
+2026-02-27 13:46:36,573 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Configure stats pid to 1829233
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Loading settings from environment variables: {}
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-02-27 13:46:36,574 WARNING MainThread:1829233 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_setup.py:_flush():77] Applying login settings: {}
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_134636-lxvmjzq8/logs/debug.log
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/wandb/run-20260227_134636-lxvmjzq8/logs/debug-internal.log
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:init():608] calling init triggers
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 2e-05, 'batch_size': 64, 'max_epochs': -1, 'max_steps': 700000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft_latent', 'precomputed_feature_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft/vae_features', 'use_explicit_memory_frames': False, 'n_frames_valid': 700, 'angle_range': 110, 'pos_range': 8, 'customized_validation': True, 'add_timestamp_embedding': True}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_a_memory_pretrain', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': True, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': True, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_a_mamba', 'customized_load': True, 'seperate_load': True, 'diffusion_model_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/diffusion_only.ckpt', 'vae_path': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/vae_only.ckpt', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a/'}
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:init():658] starting backend
+2026-02-27 13:46:36,574 INFO    MainThread:1829233 [wandb_init.py:init():662] setting up manager
+2026-02-27 13:46:36,585 INFO    MainThread:1829233 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-02-27 13:46:36,592 INFO    MainThread:1829233 [wandb_init.py:init():670] backend started and connected
+2026-02-27 13:46:36,612 INFO    MainThread:1829233 [wandb_init.py:init():768] updated telemetry
+2026-02-27 13:46:36,662 INFO    MainThread:1829233 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-02-27 13:46:37,245 ERROR   MainThread:1829233 [wandb_init.py:init():827] encountered error: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:46:37,246 ERROR   MainThread:1829233 [wandb_init.py:init():1239] error in wandb.init()
+Traceback (most recent call last):
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 1229, in init
+    return wi.init()
+  File "/proj/cvl/users/x_fahkh2/envs/worldmem/lib/python3.10/site-packages/wandb/sdk/wandb_init.py", line 833, in init
+    raise error
+wandb.errors.CommError: It appears that you do not have permission to access the requested resource. Please reach out to the project owner to grant you access. If you have the correct permissions, verify that there are no issues with your networking setup.(Error 403: Forbidden)
+2026-02-27 13:46:38,381 WARNING MsgRouterThr:1829233 [router.py:message_loop():77] message_loop has been closed

wandb_128/wandb-resume.json

@@ -0,0 +1 @@
+{"run_id": "lxvmjzq8"}