Upload folder using huggingface_hub

.gitattributes

@@ -127,3 +127,29 @@ wandb_256/offline-run-20260301_015115-wiagjqyt/run-wiagjqyt.wandb filter=lfs dif
 wandb_128/offline-run-20260227_135547-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text
 wandb_128/offline-run-20260227_141732-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text
 wandb_128/offline-run-20260301_014655-lxvmjzq8/run-lxvmjzq8.wandb filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/slurmm1.0-6236.out filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/videos/validation_vis/gt/video_0_rank0_step0.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/videos/validation_vis/pred/video_0_rank0_step0.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/run-r8lvj77y.wandb filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_100_60d71dc10d2df9717a81.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_202_c115e34d7e7e810bacb3.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_304_43e452d9f13c3fa21d80.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_406_771fab084a598e73eeac.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/run-r8lvj77y.wandb filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_100_1fe33955bb82555daa66.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_202_a5b205ac52b5cd6733d9.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_304_ea40e6ac26513bb367ad.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_406_9a75eeaf5bf1a8bdbf85.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_508_822d61f7db09caf34a7b.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_610_e2d7033b64da76eefcbb.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_712_7485db894ef46b7fd1cd.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_814_a54a3bcd353b94d9c05a.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_916_8845929a37f91cb70940.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/run-r8lvj77y.wandb filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_100_f54b338468fbdf014f88.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_202_d18387c740032582f92f.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_304_de5824376d7a99c5626e.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_406_6aa48ce9f15df653beea.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_508_08e4ecba8ac3ae98be5e.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_610_30f6643ae5b235491b91.mp4 filter=lfs diff=lfs merge=lfs -text
+bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/run-r8lvj77y.wandb filter=lfs diff=lfs merge=lfs -text

bimamba_stage_b/checkpoints/epoch0_step15000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3d2f43e70550db7ca50be140ce8b2b740e1a850e7f517bb30a27df1da8e2731
+size 8250475536

bimamba_stage_b/checkpoints/epoch0_step22500.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f38303d6e7061fcb7349c0b1e07e3b32965c61a3ebca3d3dfba040ef208a79b
+size 8250475536

bimamba_stage_b/slurmm1.0-6236.out

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f68296e0d3162f1cf4161d040f7d6ff3f2f37d7ef955e9604937a9395fcfe993
+size 13830117

bimamba_stage_b/videos/validation_vis/gt/video_0_rank0_step0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81e646cd114bde529638c8d639a6b74225214d84d90c3c398d70e1717732174e
+size 10347982

bimamba_stage_b/videos/validation_vis/pred/video_0_rank0_step0.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:729105a0bddb658d844aa7ae0bcfd8384db79db8cabbb06eb1437b791699ddd6
+size 7412083

bimamba_stage_b/wandb/debug.log

@@ -0,0 +1,26 @@
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Configure stats pid to 388316
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-05 02:55:50,511 WARNING MainThread:388316 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-05 02:55:50,511 INFO    MainThread:388316 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/logs/debug.log
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/logs/debug-internal.log
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:init():608] calling init triggers
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 2, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 0.0002, 'batch_size': 12, 'max_epochs': -1, 'max_steps': 65000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 8, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_frozen_memory', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_b_mamba', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'}
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:init():658] starting backend
+2026-03-05 02:55:50,512 INFO    MainThread:388316 [wandb_init.py:init():662] setting up manager
+2026-03-05 02:55:50,528 INFO    MainThread:388316 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-05 02:55:50,534 INFO    MainThread:388316 [wandb_init.py:init():670] backend started and connected
+2026-03-05 02:55:51,281 INFO    MainThread:388316 [wandb_init.py:init():768] updated telemetry
+2026-03-05 02:55:51,336 INFO    MainThread:388316 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-05 02:55:51,341 INFO    MainThread:388316 [wandb_init.py:init():852] starting run threads in backend
+2026-03-05 02:55:58,463 INFO    MainThread:388316 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-05 02:55:58,463 INFO    MainThread:388316 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-05 02:55:58,463 INFO    MainThread:388316 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-05 02:55:58,463 INFO    MainThread:388316 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-05 02:55:58,476 INFO    MainThread:388316 [wandb_init.py:init():895] run started, returning control to user process
+2026-03-06 01:25:39,240 WARNING MsgRouterThr:388316 [router.py:message_loop():77] message_loop has been closed

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/files/wandb-metadata.json

@@ -0,0 +1,1228 @@
+{
+    "os": "Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-03-03T00:10:24.698402",
+    "startedAt": "2026-03-03T00:10:24.387169",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_b_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=false",
+        "experiment.num_nodes=1",
+        "load=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=200",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=2",
+        "+dataset.wo_updown=false",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "+dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_b_diffusion_frozen_memory",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.enable_memory_noise_curriculum=false",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=32",
+        "algorithm.diff_window_size=8",
+        "algorithm.memory_condition_length=0",
+        "algorithm.context_frames=1",
+        "+algorithm.n_tokens=8",
+        "experiment.training.batch_size=16",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=15000",
+        "experiment.validation.val_every_n_step=2500",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node102",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1162.3798571428572,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3788.063,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.632,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3783.356,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3794.094,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.513,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3809.189,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3796.873,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3564.851,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3802.046,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.1,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3398.502,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.831867218017578
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.5592002868652
+    }
+}

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 896}}

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/logs/debug-internal.log

@@ -0,0 +1,494 @@
+2026-03-03 01:10:24,440 INFO    StreamThr :3200987 [internal.py:wandb_internal():85] W&B internal server running at pid: 3200987, started at: 2026-03-03 01:10:24.437891
+2026-03-03 01:10:24,441 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status
+2026-03-03 01:10:24,470 INFO    WriterThread:3200987 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/run-r8lvj77y.wandb
+2026-03-03 01:10:24,593 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: run_start
+2026-03-03 01:10:24,647 DEBUG   HandlerThread:3200987 [system_info.py:__init__():26] System info init
+2026-03-03 01:10:24,647 DEBUG   HandlerThread:3200987 [system_info.py:__init__():41] System info init done
+2026-03-03 01:10:24,647 INFO    HandlerThread:3200987 [system_monitor.py:start():194] Starting system monitor
+2026-03-03 01:10:24,647 INFO    SystemMonitor:3200987 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-03-03 01:10:24,647 INFO    HandlerThread:3200987 [system_monitor.py:probe():214] Collecting system info
+2026-03-03 01:10:24,647 INFO    SystemMonitor:3200987 [interfaces.py:start():188] Started cpu monitoring
+2026-03-03 01:10:24,648 INFO    SystemMonitor:3200987 [interfaces.py:start():188] Started disk monitoring
+2026-03-03 01:10:24,649 INFO    SystemMonitor:3200987 [interfaces.py:start():188] Started gpu monitoring
+2026-03-03 01:10:24,649 INFO    SystemMonitor:3200987 [interfaces.py:start():188] Started memory monitoring
+2026-03-03 01:10:24,650 INFO    SystemMonitor:3200987 [interfaces.py:start():188] Started network monitoring
+2026-03-03 01:10:24,698 DEBUG   HandlerThread:3200987 [system_info.py:probe():152] Probing system
+2026-03-03 01:10:24,700 DEBUG   HandlerThread:3200987 [system_info.py:_probe_git():137] Probing git
+2026-03-03 01:10:24,761 DEBUG   HandlerThread:3200987 [system_info.py:_probe_git():145] Probing git done
+2026-03-03 01:10:24,762 DEBUG   HandlerThread:3200987 [system_info.py:probe():200] Probing system done
+2026-03-03 01:10:24,762 DEBUG   HandlerThread:3200987 [system_monitor.py:probe():223] {'os': 'Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34', 'python': '3.10.19', 'heartbeatAt': '2026-03-03T00:10:24.698402', 'startedAt': '2026-03-03T00:10:24.387169', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_b_mamba', 'algorithm=df_video_mamba3stage', '+customized_load=true', '+seperate_load=false', 'experiment.num_nodes=1', 'load=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'dataset.n_frames=200', '+dataset.n_frames_valid=200', '+dataset.angle_range=110', '+dataset.pos_range=2', '+dataset.wo_updown=false', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', '+dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_b_diffusion_frozen_memory', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=false', 'algorithm.enable_memory_noise_curriculum=false', '+algorithm.require_pose_prediction=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=32', 'algorithm.diff_window_size=8', 'algorithm.memory_condition_length=0', 'algorithm.context_frames=1', '+algorithm.n_tokens=8', 'experiment.training.batch_size=16', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=15000', 'experiment.validation.val_every_n_step=2500', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': 'f0d488aceb5425378cec21c5d35bd92f64346f51'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node102', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 1162.3798571428572, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 3788.063, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.632, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.961, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3783.356, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3797.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.961, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3794.094, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.038, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.038, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3804.513, 'min': 800.0, 'max': 3800.0}, {'current': 3809.189, 'min': 800.0, 'max': 3800.0}, {'current': 3796.873, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3564.851, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3802.046, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.1, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3398.502, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 30.58770751953125, 'used': 16.831867218017578}}, 'gpu': 'NVIDIA H200', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}], 'memory': {'total': 2015.5592002868652}}
+2026-03-03 01:10:24,762 INFO    HandlerThread:3200987 [system_monitor.py:probe():224] Finished collecting system info
+2026-03-03 01:10:24,762 INFO    HandlerThread:3200987 [system_monitor.py:probe():227] Publishing system info
+2026-03-03 01:10:24,762 DEBUG   HandlerThread:3200987 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-03-03 01:10:30,898 DEBUG   HandlerThread:3200987 [system_info.py:_save_conda():224] Saving conda packages done
+2026-03-03 01:10:30,905 INFO    HandlerThread:3200987 [system_monitor.py:probe():229] Finished publishing system info
+2026-03-03 01:10:30,924 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: keepalive
+2026-03-03 01:10:30,924 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:30,924 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:32,227 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: python_packages
+2026-03-03 01:10:34,925 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:34,926 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:39,927 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:39,927 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:44,928 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:44,929 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:49,930 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:49,931 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:54,932 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:54,932 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:10:59,934 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:10:59,934 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:04,935 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:04,936 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:09,937 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:09,937 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:14,938 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:14,939 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:19,940 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:19,941 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:24,650 DEBUG   SystemMonitor:3200987 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-03-03 01:11:24,942 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:24,942 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:29,944 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:29,944 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:34,946 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:34,946 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:39,947 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:39,948 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:44,949 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:44,950 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:49,951 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:49,951 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:54,952 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:54,953 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:11:59,954 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:11:59,954 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:04,956 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:04,956 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:09,957 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:09,958 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:14,959 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:14,959 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:19,960 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:19,961 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:24,962 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:24,962 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:29,963 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:29,964 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:34,965 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:34,966 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:39,967 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:39,968 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:44,969 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:44,970 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:49,971 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:49,971 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:54,972 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:54,973 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:12:59,975 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:12:59,975 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:04,977 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:04,977 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:09,978 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:09,979 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:14,980 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:14,980 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:19,981 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:19,982 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:24,983 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:24,983 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:29,985 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:29,985 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:34,987 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:34,987 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:39,988 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:39,989 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:44,990 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:44,991 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:49,992 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:49,992 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:54,993 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:54,993 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:13:59,995 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:13:59,995 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:04,996 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:04,997 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:09,998 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:09,998 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:15,000 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:15,000 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:20,001 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:20,002 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:25,003 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:25,004 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:30,005 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:30,005 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:35,006 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:35,007 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:40,008 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:40,009 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:45,010 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:45,010 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:50,012 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:50,012 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:14:55,013 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:14:55,014 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:00,015 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:00,015 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:05,016 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:05,017 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:10,019 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:10,019 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:15,020 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:15,021 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:20,022 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:20,023 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:25,024 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:25,025 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:30,026 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:30,026 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:35,027 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:35,028 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:40,029 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:40,030 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:45,031 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:45,032 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:50,033 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:50,033 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:15:55,035 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:15:55,035 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:00,037 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:00,038 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:05,039 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:05,040 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:10,041 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:10,041 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:15,042 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:15,043 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:20,044 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:20,044 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:25,046 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:25,046 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:30,047 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:30,048 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:35,049 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:35,050 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:40,051 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:40,052 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:45,053 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:45,054 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:50,055 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:50,056 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:52,480 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: partial_history
+2026-03-03 01:16:55,057 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:16:55,057 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:16:56,255 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: partial_history
+2026-03-03 01:17:00,059 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:00,059 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:05,060 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:05,061 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:10,062 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:10,063 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:15,064 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:15,065 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:20,066 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:20,067 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:25,068 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:25,068 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:30,069 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:30,070 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:35,071 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:35,072 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:40,073 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:40,074 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:45,075 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:45,076 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:50,077 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:50,077 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:17:55,078 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:17:55,079 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:00,080 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:00,081 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:05,082 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:05,083 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:10,084 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:10,085 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:15,086 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:15,086 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:20,088 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:20,088 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:25,089 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:25,090 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:30,091 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:30,092 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:35,093 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:35,093 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:40,095 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:40,095 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:45,096 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:45,097 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:50,098 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:50,098 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:55,100 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:55,100 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:00,101 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:00,102 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:05,103 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:05,103 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:10,105 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:10,105 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:15,106 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:15,107 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:20,108 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:20,108 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:25,110 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:25,110 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:30,112 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:30,112 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:35,113 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:35,114 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:40,115 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:40,115 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:45,116 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:45,117 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:50,118 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:50,118 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:55,119 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:55,120 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:00,121 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:00,121 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:05,122 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:05,123 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:10,124 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:10,124 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:15,125 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:15,126 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:20,127 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:20,127 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:25,128 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:25,129 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:30,130 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:30,131 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:35,132 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:35,133 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:40,134 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:40,134 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:45,135 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:45,136 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:50,137 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:50,137 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:55,138 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:55,139 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:00,140 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:00,141 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:05,142 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:05,142 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:10,144 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:10,144 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:15,145 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:15,146 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:20,147 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:20,148 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:25,149 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:25,150 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:30,151 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:30,152 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:35,153 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:35,153 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:40,154 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:40,155 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:45,156 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:45,156 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:50,158 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:50,158 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:55,159 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:55,160 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:00,161 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:00,162 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:05,163 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:05,164 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:10,165 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:10,165 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:15,166 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:15,167 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:20,168 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:20,169 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:25,170 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:25,170 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:30,172 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:30,172 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:35,173 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:35,174 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:40,175 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:40,176 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:45,177 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:45,178 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:50,179 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:50,179 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:54,658 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: partial_history
+2026-03-03 01:22:55,180 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:22:55,181 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:22:58,027 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: partial_history
+2026-03-03 01:23:00,182 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:00,183 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:05,184 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:05,185 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:10,186 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:10,186 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:15,187 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:15,188 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:20,189 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:20,190 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:25,191 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:25,191 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:30,193 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:30,193 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:35,195 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:35,195 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:40,196 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:40,197 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:45,198 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:45,199 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:50,200 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:50,201 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:23:55,202 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:23:55,202 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:00,204 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:00,204 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:05,205 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:05,206 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:10,207 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:10,208 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:15,209 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:15,209 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:20,211 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:20,211 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:25,213 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:25,213 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:30,215 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:30,215 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:35,216 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:35,217 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:40,218 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:40,219 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:45,220 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:45,220 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:50,222 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:50,222 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:24:55,223 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:24:55,224 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:00,225 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:00,226 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:05,227 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:05,228 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:10,229 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:10,229 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:15,230 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:15,231 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:20,232 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:20,232 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:21,178 DEBUG   SenderThread:3200987 [sender.py:send():391] send: exit
+2026-03-03 01:25:21,178 INFO    SenderThread:3200987 [sender.py:send_exit():598] handling exit code: 0
+2026-03-03 01:25:21,178 INFO    SenderThread:3200987 [sender.py:send_exit():600] handling runtime: 896
+2026-03-03 01:25:21,186 INFO    SenderThread:3200987 [sender.py:_save_file():1466] saving file wandb-summary.json with policy end
+2026-03-03 01:25:21,186 INFO    SenderThread:3200987 [sender.py:send_exit():606] send defer
+2026-03-03 01:25:21,186 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:21,186 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 0
+2026-03-03 01:25:21,187 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:21,187 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 0
+2026-03-03 01:25:21,187 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 1
+2026-03-03 01:25:21,187 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:21,187 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 1
+2026-03-03 01:25:21,187 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:21,187 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 1
+2026-03-03 01:25:21,187 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 2
+2026-03-03 01:25:21,187 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:21,187 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 2
+2026-03-03 01:25:21,187 INFO    HandlerThread:3200987 [system_monitor.py:finish():203] Stopping system monitor
+2026-03-03 01:25:21,187 DEBUG   SystemMonitor:3200987 [system_monitor.py:_start():179] Finished system metrics aggregation loop
+2026-03-03 01:25:21,188 DEBUG   SystemMonitor:3200987 [system_monitor.py:_start():183] Publishing last batch of metrics
+2026-03-03 01:25:21,190 INFO    HandlerThread:3200987 [interfaces.py:finish():200] Joined cpu monitor
+2026-03-03 01:25:21,190 INFO    HandlerThread:3200987 [interfaces.py:finish():200] Joined disk monitor
+2026-03-03 01:25:22,161 INFO    HandlerThread:3200987 [interfaces.py:finish():200] Joined gpu monitor
+2026-03-03 01:25:22,162 INFO    HandlerThread:3200987 [interfaces.py:finish():200] Joined memory monitor
+2026-03-03 01:25:22,162 INFO    HandlerThread:3200987 [interfaces.py:finish():200] Joined network monitor
+2026-03-03 01:25:22,162 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,162 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 2
+2026-03-03 01:25:22,162 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 3
+2026-03-03 01:25:22,163 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,163 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 3
+2026-03-03 01:25:22,164 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,165 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 3
+2026-03-03 01:25:22,165 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 4
+2026-03-03 01:25:22,165 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,165 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 4
+2026-03-03 01:25:22,165 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,165 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 4
+2026-03-03 01:25:22,165 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 5
+2026-03-03 01:25:22,166 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,166 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 5
+2026-03-03 01:25:22,166 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,166 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 5
+2026-03-03 01:25:22,166 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 6
+2026-03-03 01:25:22,166 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,166 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 6
+2026-03-03 01:25:22,167 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,167 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 6
+2026-03-03 01:25:22,167 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 7
+2026-03-03 01:25:22,167 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:25:22,167 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,167 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 7
+2026-03-03 01:25:22,167 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:25:22,168 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,168 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 7
+2026-03-03 01:25:22,168 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 8
+2026-03-03 01:25:22,168 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,168 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 8
+2026-03-03 01:25:22,168 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,168 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 8
+2026-03-03 01:25:22,168 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 9
+2026-03-03 01:25:22,169 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,169 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 9
+2026-03-03 01:25:22,169 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,169 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 9
+2026-03-03 01:25:22,169 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 10
+2026-03-03 01:25:22,169 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,169 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 10
+2026-03-03 01:25:22,169 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,169 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 10
+2026-03-03 01:25:22,169 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 11
+2026-03-03 01:25:22,169 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,169 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 11
+2026-03-03 01:25:22,170 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 11
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 12
+2026-03-03 01:25:22,170 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,170 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 12
+2026-03-03 01:25:22,170 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 12
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 13
+2026-03-03 01:25:22,170 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,170 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 13
+2026-03-03 01:25:22,170 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 13
+2026-03-03 01:25:22,170 INFO    SenderThread:3200987 [sender.py:transition_state():626] send defer: 14
+2026-03-03 01:25:22,170 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: defer
+2026-03-03 01:25:22,171 INFO    HandlerThread:3200987 [handler.py:handle_request_defer():184] handle defer: 14
+2026-03-03 01:25:22,171 DEBUG   SenderThread:3200987 [sender.py:send():391] send: final
+2026-03-03 01:25:22,171 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: defer
+2026-03-03 01:25:22,171 INFO    SenderThread:3200987 [sender.py:send_request_defer():622] handle sender defer: 14
+2026-03-03 01:25:22,171 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: poll_exit
+2026-03-03 01:25:22,171 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: poll_exit
+2026-03-03 01:25:22,172 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: server_info
+2026-03-03 01:25:22,172 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: server_info
+2026-03-03 01:25:22,172 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: poll_exit
+2026-03-03 01:25:22,172 DEBUG   SenderThread:3200987 [sender.py:send_request():418] send_request: poll_exit
+2026-03-03 01:25:22,172 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: get_summary
+2026-03-03 01:25:22,172 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: sampled_history
+2026-03-03 01:25:22,173 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: internal_messages
+2026-03-03 01:25:22,173 INFO    MainThread:3200987 [wandb_run.py:_footer_history_summary_info():4088] rendering history
+2026-03-03 01:25:22,173 INFO    MainThread:3200987 [wandb_run.py:_footer_history_summary_info():4120] rendering summary
+2026-03-03 01:25:22,173 DEBUG   HandlerThread:3200987 [handler.py:handle_request():158] handle_request: shutdown
+2026-03-03 01:25:22,173 INFO    HandlerThread:3200987 [handler.py:finish():884] shutting down handler
+2026-03-03 01:25:23,172 INFO    WriterThread:3200987 [datastore.py:close():296] close: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/run-r8lvj77y.wandb
+2026-03-03 01:25:23,172 INFO    SenderThread:3200987 [sender.py:finish():1631] shutting down sender

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/logs/debug.log

@@ -0,0 +1,26 @@
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Configure stats pid to 3194679
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-03 01:10:24,417 WARNING MainThread:3194679 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/logs/debug.log
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/logs/debug-internal.log
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:init():608] calling init triggers
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 1, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 0.0001, 'batch_size': 16, 'max_epochs': -1, 'max_steps': 15000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 2, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_frozen_memory', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt', 'name': 'train_stage_b_mamba', 'customized_load': True, 'seperate_load': False, 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'}
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:init():658] starting backend
+2026-03-03 01:10:24,417 INFO    MainThread:3194679 [wandb_init.py:init():662] setting up manager
+2026-03-03 01:10:24,436 INFO    MainThread:3194679 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-03 01:10:24,445 INFO    MainThread:3194679 [wandb_init.py:init():670] backend started and connected
+2026-03-03 01:10:24,469 INFO    MainThread:3194679 [wandb_init.py:init():768] updated telemetry
+2026-03-03 01:10:24,536 INFO    MainThread:3194679 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-03 01:10:24,541 INFO    MainThread:3194679 [wandb_init.py:init():852] starting run threads in backend
+2026-03-03 01:10:32,226 INFO    MainThread:3194679 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-03 01:10:32,227 INFO    MainThread:3194679 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-03 01:10:32,227 INFO    MainThread:3194679 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-03 01:10:32,227 INFO    MainThread:3194679 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-03 01:10:32,240 INFO    MainThread:3194679 [wandb_init.py:init():895] run started, returning control to user process
+2026-03-03 01:25:23,179 WARNING MsgRouterThr:3194679 [router.py:message_loop():77] message_loop has been closed

bimamba_stage_b/wandb/offline-run-20260303_011024-r8lvj77y/run-r8lvj77y.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a50d1aa995c93068c6040dc74dc46112ac73dacf93862df3f2c9353b7ae810d7
+size 303331

bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/files/wandb-metadata.json

@@ -0,0 +1,1228 @@
+{
+    "os": "Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-03-03T00:18:04.014738",
+    "startedAt": "2026-03-03T00:18:03.691949",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_b_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=false",
+        "experiment.num_nodes=2",
+        "load=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=200",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=2",
+        "+dataset.wo_updown=false",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "+dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_b_diffusion_frozen_memory",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.enable_memory_noise_curriculum=false",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=32",
+        "algorithm.diff_window_size=8",
+        "algorithm.memory_condition_length=0",
+        "algorithm.context_frames=1",
+        "+algorithm.n_tokens=8",
+        "experiment.training.batch_size=16",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=15000",
+        "experiment.validation.val_every_n_step=2500",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node103",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1055.1934017857143,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3690.368,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.544,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3796.415,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3790.362,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3787.412,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3534.874,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.099,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3595.027,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3778.526,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.151,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3795.453,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.83237075805664
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.559238433838
+    }
+}

bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/logs/debug-internal.log

@@ -0,0 +1,119 @@
+2026-03-03 01:18:03,747 INFO    StreamThr :1071144 [internal.py:wandb_internal():85] W&B internal server running at pid: 1071144, started at: 2026-03-03 01:18:03.744955
+2026-03-03 01:18:03,748 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status
+2026-03-03 01:18:03,779 INFO    WriterThread:1071144 [datastore.py:open_for_write():87] open: /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/run-r8lvj77y.wandb
+2026-03-03 01:18:03,901 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: run_start
+2026-03-03 01:18:03,953 DEBUG   HandlerThread:1071144 [system_info.py:__init__():26] System info init
+2026-03-03 01:18:03,953 DEBUG   HandlerThread:1071144 [system_info.py:__init__():41] System info init done
+2026-03-03 01:18:03,953 INFO    HandlerThread:1071144 [system_monitor.py:start():194] Starting system monitor
+2026-03-03 01:18:03,954 INFO    SystemMonitor:1071144 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2026-03-03 01:18:03,954 INFO    HandlerThread:1071144 [system_monitor.py:probe():214] Collecting system info
+2026-03-03 01:18:03,955 INFO    SystemMonitor:1071144 [interfaces.py:start():188] Started cpu monitoring
+2026-03-03 01:18:03,955 INFO    SystemMonitor:1071144 [interfaces.py:start():188] Started disk monitoring
+2026-03-03 01:18:03,956 INFO    SystemMonitor:1071144 [interfaces.py:start():188] Started gpu monitoring
+2026-03-03 01:18:03,957 INFO    SystemMonitor:1071144 [interfaces.py:start():188] Started memory monitoring
+2026-03-03 01:18:03,958 INFO    SystemMonitor:1071144 [interfaces.py:start():188] Started network monitoring
+2026-03-03 01:18:04,014 DEBUG   HandlerThread:1071144 [system_info.py:probe():152] Probing system
+2026-03-03 01:18:04,017 DEBUG   HandlerThread:1071144 [system_info.py:_probe_git():137] Probing git
+2026-03-03 01:18:04,075 DEBUG   HandlerThread:1071144 [system_info.py:_probe_git():145] Probing git done
+2026-03-03 01:18:04,075 DEBUG   HandlerThread:1071144 [system_info.py:probe():200] Probing system done
+2026-03-03 01:18:04,075 DEBUG   HandlerThread:1071144 [system_monitor.py:probe():223] {'os': 'Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34', 'python': '3.10.19', 'heartbeatAt': '2026-03-03T00:18:04.014738', 'startedAt': '2026-03-03T00:18:03.691949', 'docker': None, 'cuda': None, 'args': ('+name=train_stage_b_mamba', 'algorithm=df_video_mamba3stage', '+customized_load=true', '+seperate_load=false', 'experiment.num_nodes=2', 'load=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt', 'dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'dataset.n_frames=200', '+dataset.n_frames_valid=200', '+dataset.angle_range=110', '+dataset.pos_range=2', '+dataset.wo_updown=false', '+dataset.customized_validation=true', '+dataset.add_timestamp_embedding=true', '+dataset.use_explicit_memory_frames=false', 'algorithm.training_stage=stage_b_diffusion_frozen_memory', 'algorithm.use_mamba_memory_pipeline=true', 'algorithm.use_oracle_pose_eval=false', 'algorithm.enable_memory_noise_curriculum=false', '+algorithm.require_pose_prediction=false', '+algorithm.use_memory_attention=false', '+algorithm.relative_embedding=false', '+algorithm.memory_retrieval_topk=32', 'algorithm.diff_window_size=8', 'algorithm.memory_condition_length=0', 'algorithm.context_frames=1', '+algorithm.n_tokens=8', 'experiment.training.batch_size=16', 'experiment.training.checkpointing.every_n_train_steps=2500', 'experiment.training.max_steps=15000', 'experiment.validation.val_every_n_step=2500', '+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'), 'state': 'running', 'program': '-m main', 'codePathLocal': None, 'git': {'remote': 'https://huggingface.co/BonanDing/WorldMem_Repro', 'commit': 'f0d488aceb5425378cec21c5d35bd92f64346f51'}, 'email': 'fahad.khan@liu.se', 'root': '/proj/cvl/users/x_fahkh2/WorldMem_Repro', 'host': 'node103', 'username': 'x_fahkh', 'executable': '/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 1055.1934017857143, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 3690.368, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3798.544, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3796.415, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.038, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3790.362, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3787.412, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3534.874, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3804.099, 'min': 800.0, 'max': 3800.0}, {'current': 3595.027, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3778.526, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3797.151, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 3795.453, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}, {'current': 800.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 30.58770751953125, 'used': 16.83237075805664}}, 'gpu': 'NVIDIA H200', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}, {'name': 'NVIDIA H200', 'memory_total': 150754820096}], 'memory': {'total': 2015.559238433838}}
+2026-03-03 01:18:04,075 INFO    HandlerThread:1071144 [system_monitor.py:probe():224] Finished collecting system info
+2026-03-03 01:18:04,075 INFO    HandlerThread:1071144 [system_monitor.py:probe():227] Publishing system info
+2026-03-03 01:18:04,076 DEBUG   HandlerThread:1071144 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2026-03-03 01:18:10,113 DEBUG   HandlerThread:1071144 [system_info.py:_save_conda():224] Saving conda packages done
+2026-03-03 01:18:10,118 INFO    HandlerThread:1071144 [system_monitor.py:probe():229] Finished publishing system info
+2026-03-03 01:18:10,135 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:10,135 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: keepalive
+2026-03-03 01:18:10,135 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:11,457 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: python_packages
+2026-03-03 01:18:14,136 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:14,136 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:19,137 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:19,138 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:24,139 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:24,140 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:29,141 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:29,141 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:34,142 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:34,143 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:39,144 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:39,145 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:44,146 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:44,147 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:49,147 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:49,148 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:54,149 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:54,150 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:18:59,151 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:18:59,152 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:03,959 DEBUG   SystemMonitor:1071144 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2026-03-03 01:19:04,153 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:04,153 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:09,155 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:09,156 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:14,157 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:14,157 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:19,158 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:19,159 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:24,160 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:24,161 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:29,162 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:29,163 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:34,164 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:34,164 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:39,165 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:39,166 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:44,167 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:44,168 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:49,169 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:49,170 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:54,171 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:54,172 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:19:59,173 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:19:59,173 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:04,174 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:04,175 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:09,176 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:09,177 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:14,178 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:14,179 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:19,180 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:19,181 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:24,182 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:24,182 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:29,183 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:29,184 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:34,185 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:34,186 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:39,187 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:39,188 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:44,189 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:44,189 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:49,190 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:49,190 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:54,191 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:54,191 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:20:59,192 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:20:59,193 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:04,193 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:04,194 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:09,194 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:09,195 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:14,195 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:14,196 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:19,196 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:19,197 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:24,198 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:24,198 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:29,199 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:29,199 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:34,200 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:34,200 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:39,201 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:39,201 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:44,202 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:44,202 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:49,203 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:49,203 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report
+2026-03-03 01:21:54,204 DEBUG   HandlerThread:1071144 [handler.py:handle_request():158] handle_request: status_report
+2026-03-03 01:21:54,204 DEBUG   SenderThread:1071144 [sender.py:send_request():418] send_request: status_report

bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/logs/debug.log

@@ -0,0 +1,25 @@
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Configure stats pid to 1062706
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-03 01:18:03,720 WARNING MainThread:1062706 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/logs/debug.log
+2026-03-03 01:18:03,720 INFO    MainThread:1062706 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_011803-r8lvj77y/logs/debug-internal.log
+2026-03-03 01:18:03,721 INFO    MainThread:1062706 [wandb_init.py:init():608] calling init triggers
+2026-03-03 01:18:03,721 INFO    MainThread:1062706 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 2, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 0.0001, 'batch_size': 16, 'max_epochs': -1, 'max_steps': 15000, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 2, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_frozen_memory', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt', 'name': 'train_stage_b_mamba', 'customized_load': True, 'seperate_load': False, 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'}
+2026-03-03 01:18:03,721 INFO    MainThread:1062706 [wandb_init.py:init():658] starting backend
+2026-03-03 01:18:03,721 INFO    MainThread:1062706 [wandb_init.py:init():662] setting up manager
+2026-03-03 01:18:03,743 INFO    MainThread:1062706 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-03 01:18:03,752 INFO    MainThread:1062706 [wandb_init.py:init():670] backend started and connected
+2026-03-03 01:18:03,779 INFO    MainThread:1062706 [wandb_init.py:init():768] updated telemetry
+2026-03-03 01:18:03,844 INFO    MainThread:1062706 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-03 01:18:03,849 INFO    MainThread:1062706 [wandb_init.py:init():852] starting run threads in backend
+2026-03-03 01:18:11,456 INFO    MainThread:1062706 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-03 01:18:11,457 INFO    MainThread:1062706 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-03 01:18:11,457 INFO    MainThread:1062706 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-03 01:18:11,457 INFO    MainThread:1062706 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-03 01:18:11,470 INFO    MainThread:1062706 [wandb_init.py:init():895] run started, returning control to user process

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_100_60d71dc10d2df9717a81.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60d71dc10d2df9717a8183af1c4571bceb6b1819be531f30538591c551b1e440
+size 4576746

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_202_c115e34d7e7e810bacb3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c115e34d7e7e810bacb34433a2eef3d8aeda0932970a6f54911a6fd5115e77ab
+size 4561732

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_304_43e452d9f13c3fa21d80.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43e452d9f13c3fa21d8073abae40bcda0da925a147a168641d2cd1031597c73a
+size 4563665

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/media/videos/validation_vis/video_0_406_771fab084a598e73eeac.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:771fab084a598e73eeac8cc0c146884fa794db2698339b260d2cd3f97b8ff5eb
+size 4559600

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/wandb-metadata.json

@@ -0,0 +1,1228 @@
+{
+    "os": "Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-03-03T00:26:45.105560",
+    "startedAt": "2026-03-03T00:26:44.822124",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_b_mamba",
+        "algorithm=df_video_mamba3stage",
+        "+customized_load=true",
+        "+seperate_load=false",
+        "experiment.num_nodes=2",
+        "load=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=200",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=2",
+        "+dataset.wo_updown=false",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "+dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_b_diffusion_frozen_memory",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.enable_memory_noise_curriculum=false",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=32",
+        "algorithm.diff_window_size=8",
+        "algorithm.memory_condition_length=0",
+        "algorithm.context_frames=1",
+        "+algorithm.n_tokens=8",
+        "experiment.training.batch_size=12",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=11250",
+        "experiment.validation.val_every_n_step=2500",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node107",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1093.1603303571428,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3773.938,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3791.548,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3414.228,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.482,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.543,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3792.878,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3781.34,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.534,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.97,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.056,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3812.277,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.55,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3802.132,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.835540771484375
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.5591926574707
+    }
+}

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 60066}}

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/logs/debug.log

@@ -0,0 +1,26 @@
+2026-03-03 01:26:44,851 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Configure stats pid to 3965936
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-03 01:26:44,852 WARNING MainThread:3965936 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/logs/debug.log
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/logs/debug-internal.log
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:init():608] calling init triggers
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 2, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 0.0002, 'batch_size': 12, 'max_epochs': -1, 'max_steps': 11250, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 2, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_frozen_memory', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_a_256/checkpoints/stage_1.ckpt', 'name': 'train_stage_b_mamba', 'customized_load': True, 'seperate_load': False, 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'}
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:init():658] starting backend
+2026-03-03 01:26:44,852 INFO    MainThread:3965936 [wandb_init.py:init():662] setting up manager
+2026-03-03 01:26:44,871 INFO    MainThread:3965936 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-03 01:26:44,879 INFO    MainThread:3965936 [wandb_init.py:init():670] backend started and connected
+2026-03-03 01:26:44,902 INFO    MainThread:3965936 [wandb_init.py:init():768] updated telemetry
+2026-03-03 01:26:44,962 INFO    MainThread:3965936 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-03 01:26:44,967 INFO    MainThread:3965936 [wandb_init.py:init():852] starting run threads in backend
+2026-03-03 01:26:53,664 INFO    MainThread:3965936 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-03 01:26:53,665 INFO    MainThread:3965936 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-03 01:26:53,665 INFO    MainThread:3965936 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-03 01:26:53,665 INFO    MainThread:3965936 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-03 01:26:53,684 INFO    MainThread:3965936 [wandb_init.py:init():895] run started, returning control to user process
+2026-03-03 18:07:53,097 WARNING MsgRouterThr:3965936 [router.py:message_loop():77] message_loop has been closed

bimamba_stage_b/wandb/offline-run-20260303_012644-r8lvj77y/run-r8lvj77y.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e40bc45611e2b6934a7a8de88507e69268490d9cc858018deb919a37126e5f95
+size 21611244

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_100_1fe33955bb82555daa66.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fe33955bb82555daa66440e58f688567a3e1da558e27d40c104e45d295dc612
+size 3012651

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_202_a5b205ac52b5cd6733d9.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5b205ac52b5cd6733d966c0fd6b98cccc233f7dd0e5b762c8abf41ffdcc2e66
+size 3007313

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_304_ea40e6ac26513bb367ad.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea40e6ac26513bb367ad34a6468a000fd65259f18fa46d4e7cd05afe94804f95
+size 3424747

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_406_9a75eeaf5bf1a8bdbf85.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a75eeaf5bf1a8bdbf85edd0516a3d92ec4008ae9753ddcc2325dfc536f717b7
+size 3053982

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_508_822d61f7db09caf34a7b.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:822d61f7db09caf34a7b78a6901910d74275a411eb2a08856a49850cb106f654
+size 3203336

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_610_e2d7033b64da76eefcbb.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2d7033b64da76eefcbb6fa19f7142c722bcb3c8829d981d1d8725dfbd996ba8
+size 2832997

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_712_7485db894ef46b7fd1cd.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7485db894ef46b7fd1cdee3e36b234d4ac1432341ad4e955d44840ea1b024522
+size 3204384

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_814_a54a3bcd353b94d9c05a.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a54a3bcd353b94d9c05a0ed46f4ed8f887a0526fa0bec368b17c3ff44e3832d2
+size 2804054

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/media/videos/validation_vis/video_0_916_8845929a37f91cb70940.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8845929a37f91cb70940b51013c7342c3305953891403122076498ec04749e9d
+size 2733933

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/wandb-metadata.json

@@ -0,0 +1,1225 @@
+{
+    "os": "Linux-5.14.0-611.30.1.el9_7.x86_64-x86_64-with-glibc2.34",
+    "python": "3.10.19",
+    "heartbeatAt": "2026-03-03T17:08:56.002525",
+    "startedAt": "2026-03-03T17:08:55.733456",
+    "docker": null,
+    "cuda": null,
+    "args": [
+        "+name=train_stage_b_mamba",
+        "algorithm=df_video_mamba3stage",
+        "experiment.num_nodes=2",
+        "dataset.save_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft",
+        "dataset.n_frames=200",
+        "+dataset.n_frames_valid=200",
+        "+dataset.angle_range=110",
+        "+dataset.pos_range=8",
+        "+dataset.wo_updown=false",
+        "+dataset.customized_validation=true",
+        "+dataset.add_timestamp_embedding=true",
+        "+dataset.use_explicit_memory_frames=false",
+        "algorithm.training_stage=stage_b_diffusion_frozen_memory",
+        "algorithm.use_mamba_memory_pipeline=true",
+        "algorithm.use_oracle_pose_eval=false",
+        "algorithm.enable_memory_noise_curriculum=false",
+        "+algorithm.require_pose_prediction=false",
+        "+algorithm.use_memory_attention=false",
+        "+algorithm.relative_embedding=false",
+        "+algorithm.memory_retrieval_topk=32",
+        "algorithm.diff_window_size=8",
+        "algorithm.memory_condition_length=0",
+        "algorithm.context_frames=1",
+        "+algorithm.n_tokens=8",
+        "experiment.training.batch_size=12",
+        "experiment.training.checkpointing.every_n_train_steps=2500",
+        "experiment.training.max_steps=22500",
+        "experiment.validation.val_every_n_step=2500",
+        "+output_dir=/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/"
+    ],
+    "state": "running",
+    "program": "-m main",
+    "codePathLocal": null,
+    "git": {
+        "remote": "https://huggingface.co/BonanDing/WorldMem_Repro",
+        "commit": "f0d488aceb5425378cec21c5d35bd92f64346f51"
+    },
+    "email": "fahad.khan@liu.se",
+    "root": "/proj/cvl/users/x_fahkh2/WorldMem_Repro",
+    "host": "node107",
+    "username": "x_fahkh",
+    "executable": "/proj/cvl/users/x_fahkh2/envs/worldmem/bin/python",
+    "cpu_count": 112,
+    "cpu_count_logical": 224,
+    "cpu_freq": {
+        "current": 1082.7699553571426,
+        "min": 800.0,
+        "max": 3800.0
+    },
+    "cpu_freq_per_core": [
+        {
+            "current": 3795.605,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3801.352,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3797.96,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3539.782,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3798.151,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.038,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.004,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3781.096,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3802.091,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3803.353,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3795.657,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3804.274,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3799.961,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 3800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        },
+        {
+            "current": 800.0,
+            "min": 800.0,
+            "max": 3800.0
+        }
+    ],
+    "disk": {
+        "/": {
+            "total": 30.58770751953125,
+            "used": 16.83456039428711
+        }
+    },
+    "gpu": "NVIDIA H200",
+    "gpu_count": 8,
+    "gpu_devices": [
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        },
+        {
+            "name": "NVIDIA H200",
+            "memory_total": 150754820096
+        }
+    ],
+    "memory": {
+        "total": 2015.5591926574707
+    }
+}

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 117949}}

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/logs/debug.log

@@ -0,0 +1,26 @@
+2026-03-03 18:08:55,759 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Current SDK version is 0.17.9
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Configure stats pid to 2764885
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Loading settings from /home/x_fahkh/.config/wandb/settings
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Loading settings from /proj/cvl/users/x_fahkh2/WorldMem_Repro/wandb/settings
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Loading settings from environment variables: {'disabled': 'true'}
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Applying setup settings: {'_disable_service': False}
+2026-03-03 18:08:55,760 WARNING MainThread:2764885 [wandb_setup.py:_flush():77] Could not find program at -m main
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_setup.py:_flush():77] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m main'}
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:_log_setup():524] Logging user logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/logs/debug.log
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:_log_setup():525] Logging internal logs to /proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/logs/debug-internal.log
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:init():608] calling init triggers
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:init():615] wandb.init called with sweep_config: {}
+config: {'experiment': {'debug': '${debug}', 'tasks': ['training'], 'num_nodes': 2, 'training': {'precision': '16-mixed', 'compile': False, 'lr': 0.0002, 'batch_size': 12, 'max_epochs': -1, 'max_steps': 22500, 'max_time': None, 'data': {'num_workers': 4, 'shuffle': True}, 'optim': {'accumulate_grad_batches': 1, 'gradient_clip_val': 1.0}, 'checkpointing': {'every_n_train_steps': 2500, 'every_n_epochs': None, 'train_time_interval': None, 'enable_version_counter': False}}, 'validation': {'precision': '16-mixed', 'compile': False, 'batch_size': 4, 'val_every_n_step': 2500, 'val_every_n_epoch': None, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'test': {'precision': '16-mixed', 'compile': False, 'batch_size': 1, 'limit_batch': 1, 'inference_mode': False, 'data': {'num_workers': 4, 'shuffle': False}}, 'logging': {'metrics': None}, '_name': 'exp_video'}, 'dataset': {'debug': '${debug}', 'metadata': 'data/${dataset.name}/metadata.json', 'data_mean': 0.5, 'data_std': 0.5, 'save_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/datasets/minecraft', 'n_frames': 200, 'context_length': 1, 'resolution': 128, 'observation_shape': [3, '${dataset.resolution}', '${dataset.resolution}'], 'external_cond_dim': 0, 'validation_multiplier': 1, 'frame_skip': 1, 'action_cond_dim': 25, '_name': 'video_minecraft', 'n_frames_valid': 200, 'angle_range': 110, 'pos_range': 8, 'wo_updown': False, 'customized_validation': True, 'add_timestamp_embedding': True, 'use_explicit_memory_frames': False}, 'algorithm': {'debug': '${debug}', 'lr': '${experiment.training.lr}', 'x_shape': '${dataset.observation_shape}', 'frame_stack': 1, 'frame_skip': '${dataset.frame_skip}', 'data_mean': '${dataset.data_mean}', 'data_std': '${dataset.data_std}', 'external_cond_dim': 0, 'context_frames': 1, 'weight_decay': 0.002, 'warmup_steps': 1000, 'optimizer_beta': [0.9, 0.99], 'uncertainty_scale': 1, 'guidance_scale': 0.0, 'chunk_size': 1, 'scheduling_matrix': 'autoregressive', 'noise_level': 'random_all', 'causal': True, 'diffusion': {'objective': 'pred_v', 'beta_schedule': 'sigmoid', 'schedule_fn_kwargs': {}, 'clip_noise': 20.0, 'use_snr': False, 'use_cum_snr': False, 'use_fused_snr': True, 'snr_clip': 5.0, 'cum_snr_decay': 0.96, 'timesteps': 1000, 'sampling_timesteps': 20, 'ddim_sampling_eta': 0.0, 'stabilization_level': 15, 'architecture': {'network_size': 64, 'attn_heads': 4, 'attn_dim_head': 64, 'dim_mults': [1, 2, 4, 8], 'resolution': '${dataset.resolution}', 'attn_resolutions': [16, 32, 64, 128], 'use_init_temporal_attn': True, 'use_linear_attn': True, 'time_emb_type': 'rotary'}}, 'n_frames': '${dataset.n_frames}', 'metadata': '${dataset.metadata}', 'action_cond_dim': 25, 'use_plucker': True, 'memory_condition_length': 0, 'log_video': True, 'use_compressed_causal_memory': False, 'compressed_memory_dim': 256, 'compressed_memory_slots': 64, 'compressed_memory_topk': 4, 'use_mamba_memory_pipeline': True, 'training_stage': 'stage_b_diffusion_frozen_memory', 'stage_c_memory_aux_weight': 0.1, 'diff_window_size': 8, 'memory_gap_aux_weight': 0.1, 'memory_gap_aux_weight_init': 0.5, 'memory_gap_aux_weight_final': 0.1, 'memory_gap_aux_decay_fraction': 0.3, 'memory_gap_aux_decay_steps': 0, 'memory_gap_aux_exp_k': 5.0, 'use_precomputed_features': False, 'mamba_latent_channels': 16, 'mamba_model_dim': 256, 'mamba_depth': 4, 'mamba_cond_dim': 256, 'mamba_d_state': 16, 'mamba_d_conv': 4, 'mamba_expand': 2, 'allow_mamba_fallback': False, 'strict_causal_training': True, 'strict_causal_evaluation': True, 'use_oracle_pose_eval': False, 'enable_memory_noise_curriculum': False, 'curriculum_phase_boundaries': [0.2, 0.7], 'curriculum_noise_ranges': [[600, 1000], [200, 900], [0, 400]], 'curriculum_horizons': [8, 64, 256], '_name': 'df_video_mamba3stage', 'require_pose_prediction': False, 'use_memory_attention': False, 'relative_embedding': False, 'memory_retrieval_topk': 32, 'n_tokens': 8}, 'debug': False, 'wandb': {'entity': 'turlin', 'project': 'worldmem', 'mode': 'online'}, 'resume': None, 'load': None, 'name': 'train_stage_b_mamba', 'output_dir': '/proj/cvl/users/x_fahkh2/WorldMem_Repro/checkpoints/bimamba_stage_b/'}
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:init():658] starting backend
+2026-03-03 18:08:55,760 INFO    MainThread:2764885 [wandb_init.py:init():662] setting up manager
+2026-03-03 18:08:55,774 INFO    MainThread:2764885 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-03-03 18:08:55,781 INFO    MainThread:2764885 [wandb_init.py:init():670] backend started and connected
+2026-03-03 18:08:55,805 INFO    MainThread:2764885 [wandb_init.py:init():768] updated telemetry
+2026-03-03 18:08:55,852 INFO    MainThread:2764885 [wandb_init.py:init():801] communicating run to backend with 90.0 second timeout
+2026-03-03 18:08:55,857 INFO    MainThread:2764885 [wandb_init.py:init():852] starting run threads in backend
+2026-03-03 18:09:02,899 INFO    MainThread:2764885 [wandb_run.py:_console_start():2465] atexit reg
+2026-03-03 18:09:02,899 INFO    MainThread:2764885 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-03-03 18:09:02,900 INFO    MainThread:2764885 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-03-03 18:09:02,900 INFO    MainThread:2764885 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-03-03 18:09:02,910 INFO    MainThread:2764885 [wandb_init.py:init():895] run started, returning control to user process
+2026-03-05 02:54:47,800 WARNING MsgRouterThr:2764885 [router.py:message_loop():77] message_loop has been closed

bimamba_stage_b/wandb/offline-run-20260303_180855-r8lvj77y/run-r8lvj77y.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1489ed00637aa0bf217fef49701adb03e28ea6b65c0bb95952c8d293c876008e
+size 42790922

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/conda-environment.yaml

@@ -0,0 +1,4 @@
+name: worldmem
+channels:
+  - defaults
+prefix: /home/x_fahkh/.conda/envs/worldmem

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_100_f54b338468fbdf014f88.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54b338468fbdf014f8815a9470e1c2999e927c0287fa673a6bfd445eb063f95
+size 4219563

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_202_d18387c740032582f92f.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d18387c740032582f92f1a147bdf46b664be42e8121300772bcc1c59456fc5e0
+size 6698430

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_304_de5824376d7a99c5626e.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de5824376d7a99c5626e0ace3afea5f319fe9759ac0c513c6d1e11a115d59de1
+size 4617915

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_406_6aa48ce9f15df653beea.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aa48ce9f15df653beea71a757f27618fa8f559387d4861d798d57ce83309bcd
+size 4209347

bimamba_stage_b/wandb/offline-run-20260305_025550-r8lvj77y/files/media/videos/validation_vis/video_0_508_08e4ecba8ac3ae98be5e.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08e4ecba8ac3ae98be5edca9f65334510d3ce6c2c5a098b9b6643fc25292d9bf
+size 5806106