Add files using upload-large-folder tool

.gitattributes

@@ -70,3 +70,16 @@ videos/round002/f4f8079f6637_k000_a403ac96.mp4 filter=lfs diff=lfs merge=lfs -te
 videos/round002/7ebc8b336779_k001_585674db.mp4 filter=lfs diff=lfs merge=lfs -text
 videos/round002/f4f8079f6637_k001_a40bb712.mp4 filter=lfs diff=lfs merge=lfs -text
 videos/round002/f4f8079f6637_k001_45c1d63e.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_ddafd33c_CLEAN_ms1.0641.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_32666f10_CLEAN_ms1.1054.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_89927d91_CLEAN_ms1.3450.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k001_62e8daff_FAIL_ms1.0768.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_140f07dc_CLEAN_ms1.1054.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k001_6615c2c4_CLEAN_ms1.1069.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_2d3023c7_CLEAN_ms1.0881.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_4dd03d13_CLEAN_ms1.0985.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_1c91e481_CLEAN_ms1.0826.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_c304d5a9_CLEAN_ms1.0826.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_98e9a7bc_CLEAN_ms1.3450.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k001_65f6114e_CLEAN_ms1.0744.mp4 filter=lfs diff=lfs merge=lfs -text
+reward_debug/round000/1b67584bff87_k000_6843a418_CLEAN_ms1.0641.mp4 filter=lfs diff=lfs merge=lfs -text

reward_debug/round000/1b67584bff87_k000_140f07dc_CLEAN_ms1.1054.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbf507778ce75b6bb56bab71c7494c67addb54ce20d32ef56bcb822da25c3ab4
+size 2361972

reward_debug/round000/1b67584bff87_k000_1c91e481_CLEAN_ms1.0826.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b9bb67e3c7ac5da546a5ca12c4449a4effeb623d45eaf344a42e7ea8261152f
+size 2339614

reward_debug/round000/1b67584bff87_k000_2d3023c7_CLEAN_ms1.0881.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab2fab2f0908c2fb038d679cb3415aabc412b67e8ce4ac2abfcadab5a4c489cd
+size 2360584

reward_debug/round000/1b67584bff87_k000_32666f10_CLEAN_ms1.1054.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbf507778ce75b6bb56bab71c7494c67addb54ce20d32ef56bcb822da25c3ab4
+size 2361972

reward_debug/round000/1b67584bff87_k000_4dd03d13_CLEAN_ms1.0985.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:788fdbf6c6081f1182cf9b162937f6dfd58e217d9691989f7b277e6c3732936f
+size 2356905

reward_debug/round000/1b67584bff87_k000_6843a418_CLEAN_ms1.0641.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2473aab17063a10360651d35b4785e4993ececb5ccce50874616ee87a90c7c70
+size 2392799

reward_debug/round000/1b67584bff87_k000_89927d91_CLEAN_ms1.3450.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fa9070952b043c3da2268df805a52ff3df7058b04e83d9b7e00e2285871fec1
+size 2400922

reward_debug/round000/1b67584bff87_k000_98e9a7bc_CLEAN_ms1.3450.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fa9070952b043c3da2268df805a52ff3df7058b04e83d9b7e00e2285871fec1
+size 2400922

reward_debug/round000/1b67584bff87_k000_c304d5a9_CLEAN_ms1.0826.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b9bb67e3c7ac5da546a5ca12c4449a4effeb623d45eaf344a42e7ea8261152f
+size 2339614

reward_debug/round000/1b67584bff87_k000_ddafd33c_CLEAN_ms1.0641.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2473aab17063a10360651d35b4785e4993ececb5ccce50874616ee87a90c7c70
+size 2392799

reward_debug/round000/1b67584bff87_k001_62e8daff_FAIL_ms1.0768.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1fed03bdb01e0194771940b43a20058bd46224a3e36273f761fe78e5ff5e52a6
+size 2376223

reward_debug/round000/1b67584bff87_k001_65f6114e_CLEAN_ms1.0744.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eafe2bfbd7107ccb9fbd1b1838b70a3d7ddee382f6d04f9e8fb058f8578db96
+size 2323915

reward_debug/round000/1b67584bff87_k001_6615c2c4_CLEAN_ms1.1069.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10ee7b335cad92e16e4ce034607ffb5d7db382e1e78f381389822fbc4b21667f
+size 2350533

wandb/run-20260408_000908-i6zi4vwm/files/config.yaml

@@ -0,0 +1,145 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.11.2
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.2
+            "5": 0.18.5
+            "6": 4.46.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+K:
+    value: 16
+bsv2_check_window_frac:
+    value: 0.2
+bsv2_dup_max_frames:
+    value: 0
+bsv2_expected_grip_changes:
+    value: 6
+bsv2_idm_ckpt_path:
+    value: ckpts/vidar_ckpt/idm.pt
+bsv2_mj_hi:
+    value: 1.3
+bsv2_mj_lo:
+    value: 0.6
+bsv2_pick_thr:
+    value: 0.05
+bsv2_place_thr:
+    value: 0.05
+bsv2_prompts:
+    value:
+        - red block
+        - green block
+        - blue block
+bsv2_vertical_sep_thr:
+    value: 0.03
+bsv2_x_align_thr:
+    value: 0.025
+checkpointing_steps:
+    value: 10
+ckpt_dir:
+    value: ckpts/Wan2.2-TI2V-5B
+convert_model_dtype:
+    value: true
+dataset_json:
+    value: data/rl_train/robotwin_stack_blocks_three.json
+effective_batch_size:
+    value: 16
+epochs_per_round:
+    value: 100
+frame_num:
+    value: 121
+gradient_checkpointing:
+    value: true
+hallucination_crop_top_ratio:
+    value: 0.6667
+lambda_gripper:
+    value: 1
+learning_rate:
+    value: 1e-05
+lora_alpha:
+    value: 64
+lora_rank:
+    value: 64
+lora_target_modules:
+    value: null
+max_grad_norm:
+    value: 2
+max_per_condition:
+    value: 64
+max_samples:
+    value: -1
+neg_prompt:
+    value: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+num_ode_steps:
+    value: 20
+num_rounds:
+    value: 10
+num_train_timesteps:
+    value: 1000
+offload_model:
+    value: false
+output_dir:
+    value: data/outputs/creflow_stack_blocks_three
+pt_dir:
+    value: ckpts/vidar_ckpt/merged_vidar_lora.pt
+reset_optimizer_per_round:
+    value: false
+resume_from_lora_checkpoint:
+    value: null
+reward_backend:
+    value: blocks_stack_v2
+reward_config:
+    value: blocks_stack_v2
+sample_guide_scale:
+    value: 5
+sample_shift:
+    value: 5
+seed:
+    value: 42
+size:
+    value: 640*736
+skip_reward_debug_video:
+    value: true
+task:
+    value: ti2v-5B
+use_8bit_adam:
+    value: true
+vidar_root:
+    value: ""
+w_bad:
+    value: 1
+wandb_project:
+    value: Corrective-Reflow
+wandb_run_name:
+    value: creflow_stack_blocks_three
+weight_decay:
+    value: 0.01

wandb/run-20260408_000908-i6zi4vwm/files/output.log

@@ -0,0 +1,139 @@
+Building Wan2.2 TI2V model ... (DDP=False, world_size=1)
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 00:10:37,404 17550 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+  Gradient checkpointing enabled on 30 DiT blocks
+Trainable parameters: 94.4 M
+  Using 8-bit AdamW (bitsandbytes)
+Dataset: 10 conditions
+Encoding 10 conditions ...
+  [1/10] 6b20973f10ef... encoded
+  [2/10] cff33035b28a... encoded
+  [3/10] 7ebc8b336779... encoded
+  [4/10] 8077c679fb62... encoded
+  [5/10] 41855438a8e0... encoded
+  [6/10] a67b02f4e7ce... encoded
+  [7/10] 4568a9603f3b... encoded
+  [8/10] 1b67584bff87... encoded
+  [9/10] f4f8079f6637... encoded
+  [10/10] 1b70247c6412... encoded
+All 10 conditions encoded.
+
+Starting Corrective Reflow | rounds=10  K=16  K_local=16  epochs/round=100  effective_bs=16  lr=1e-05  conditions=10
+
+======================================================================
+ROUND 1/10
+======================================================================
+[Phase 1] ODE rollout ...
+  [CReflow rollout] condition 0/10: K=16, id=6b20973f10ef...
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1034, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 866, in main
+    rollouts = creflow_rollout(
+               ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 267, in creflow_rollout
+    x0_preds = _ode_rollout_batch(
+               ^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 139, in _ode_rollout_batch
+    outputs = dit(x_list, t=ts_batch, context=context_list, seq_len=seq_len)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 812, in forward
+    return self.get_base_model()(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 498, in forward
+    x = block(x, **kwargs)
+        ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 126, in _ckpt_fwd
+    return torch_ckpt.checkpoint(orig_fwd, *args, use_reentrant=False, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_compile.py", line 53, in inner
+    return disable_fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+    ret = function(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 241, in forward
+    e = (self.modulation.unsqueeze(0) + e).chunk(6, dim=2)
+         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 31.33 GiB. GPU 0 has a total capacity of 79.11 GiB of which 4.63 GiB is free. Process 3796 has 0 bytes memory in use. Process 8629 has 0 bytes memory in use. Including non-PyTorch memory, this process has 0 bytes memory in use. Of the allocated memory 66.44 GiB is allocated by PyTorch, and 5.42 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1034, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 866, in main
+    rollouts = creflow_rollout(
+               ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 267, in creflow_rollout
+    x0_preds = _ode_rollout_batch(
+               ^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 139, in _ode_rollout_batch
+    outputs = dit(x_list, t=ts_batch, context=context_list, seq_len=seq_len)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 812, in forward
+    return self.get_base_model()(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 498, in forward
+    x = block(x, **kwargs)
+        ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 126, in _ckpt_fwd
+    return torch_ckpt.checkpoint(orig_fwd, *args, use_reentrant=False, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_compile.py", line 53, in inner
+    return disable_fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 503, in checkpoint
+    ret = function(*args, **kwargs)
+          ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 241, in forward
+    e = (self.modulation.unsqueeze(0) + e).chunk(6, dim=2)
+         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~~
+torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 31.33 GiB. GPU 0 has a total capacity of 79.11 GiB of which 4.63 GiB is free. Process 3796 has 0 bytes memory in use. Process 8629 has 0 bytes memory in use. Including non-PyTorch memory, this process has 0 bytes memory in use. Of the allocated memory 66.44 GiB is allocated by PyTorch, and 5.42 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation.  See documentation for Memory Management  (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)

wandb/run-20260408_000908-i6zi4vwm/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T00:09:08.281832Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "912b225d838164e0178805670881ee201ad81ff0"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "477545373696"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_000908-i6zi4vwm/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":121}}

wandb/run-20260408_000908-i6zi4vwm/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Configure stats pid to 17550
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 00:09:08,247 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 00:09:08,248 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 00:09:08,248 INFO    MainThread:17550 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 00:09:08,249 INFO    MainThread:17550 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_000908-i6zi4vwm/logs/debug.log
+2026-04-08 00:09:08,251 INFO    MainThread:17550 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_000908-i6zi4vwm/logs/debug-internal.log
+2026-04-08 00:09:08,251 INFO    MainThread:17550 [wandb_init.py:init():621] calling init triggers
+2026-04-08 00:09:08,251 INFO    MainThread:17550 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 00:09:08,251 INFO    MainThread:17550 [wandb_init.py:init():671] starting backend
+2026-04-08 00:09:08,251 INFO    MainThread:17550 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 00:09:08,279 INFO    MainThread:17550 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 00:09:08,279 INFO    MainThread:17550 [wandb_init.py:init():688] backend started and connected
+2026-04-08 00:09:08,337 INFO    MainThread:17550 [wandb_init.py:init():783] updated telemetry
+2026-04-08 00:09:08,505 INFO    MainThread:17550 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 00:09:08,884 INFO    MainThread:17550 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 00:09:09,247 INFO    MainThread:17550 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 00:09:09,247 INFO    MainThread:17550 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 00:09:09,247 INFO    MainThread:17550 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 00:09:09,247 INFO    MainThread:17550 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 00:09:09,252 INFO    MainThread:17550 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 00:11:09,780 WARNING MsgRouterThr:17550 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_001541-g1xdtpwt/files/output.log

@@ -0,0 +1,286 @@
+Building Wan2.2 TI2V model ... (DDP=True, world_size=8)
+/usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  warnings.warn(  # warn only once
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 00:16:58,025 18968 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+  Gradient checkpointing enabled on 30 DiT blocks
+Trainable parameters: 94.4 M
+  Using 8-bit AdamW (bitsandbytes)
+Dataset: 10 conditions
+Encoding 10 conditions ...
+  [1/10] 6b20973f10ef... encoded
+  [2/10] cff33035b28a... encoded
+  [3/10] 7ebc8b336779... encoded
+  [4/10] 8077c679fb62... encoded
+  [5/10] 41855438a8e0... encoded
+  [6/10] a67b02f4e7ce... encoded
+  [7/10] 4568a9603f3b... encoded
+  [8/10] 1b67584bff87... encoded
+  [9/10] f4f8079f6637... encoded
+  [10/10] 1b70247c6412... encoded
+All 10 conditions encoded.
+
+Starting Corrective Reflow | rounds=10  K=16  K_local=2  epochs/round=100  effective_bs=16  lr=1e-05  conditions=10
+
+======================================================================
+ROUND 1/10
+======================================================================
+[Phase 1] ODE rollout ...
+  [CReflow rollout] condition 0/10: K=2, id=6b20973f10ef...
+    rollout: 2 videos in 56.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.17it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:12<00:00,  9.53it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.55it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.19it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.09it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.96it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 72.76it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.60it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.24it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.41it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.98it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.12it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 1/10: K=2, id=cff33035b28a...
+    rollout: 2 videos in 56.9s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.44it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.01it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.92it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.90it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.73it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.73it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.54it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.75it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.42it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.50it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.13it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.15it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 2/10: K=2, id=7ebc8b336779...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.66it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.60it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.08it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.02it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.85it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.32it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.15it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.05it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.33it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.75it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.09it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 3/10: K=2, id=8077c679fb62...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.11it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.56it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.25it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.81it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 85.87it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.61it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.69it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.56it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.94it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.82it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.03it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 4/10: K=2, id=41855438a8e0...
+    rollout: 2 videos in 57.1s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.83it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.61it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.09it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.07it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.11it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.79it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.75it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.69it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.43it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.37it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.28it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.71it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 5/10: K=2, id=a67b02f4e7ce...
+    rollout: 2 videos in 57.1s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.96it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:11<00:00, 10.48it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.64it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.48it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.33it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.43it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.50it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.01it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.30it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.11it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.23it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 6/10: K=2, id=4568a9603f3b...
+    rollout: 2 videos in 57.7s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.97it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.58it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.65it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.75it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.19it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.97it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.75it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.32it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.90it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.36it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.29it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.29it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 7/10: K=2, id=1b67584bff87...
+    rollout: 2 videos in 57.4s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.97it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.02it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 76.90it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.12it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.20it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.42it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.36it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.12it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.63it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.37it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.10it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.13it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 8/10: K=2, id=f4f8079f6637...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.98it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.02it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.98it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.34it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.75it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.74it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.41it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.15it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.98it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.29it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.62it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.98it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 9/10: K=2, id=1b70247c6412...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.88it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 85.68it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.16it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.28it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.22it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.94it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.26it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.23it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.00it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.12it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.45it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] done: 20 samples, rewards = [1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 0.00, 0.00]
+  Rollout done: 20 samples, good=18, bad=2, avg_reward=0.900
+    1b67584bff87: mean=1.00, good=2/2
+    1b70247c6412: mean=0.00, good=0/2
+    41855438a8e0: mean=1.00, good=2/2
+    4568a9603f3b: mean=1.00, good=2/2
+    6b20973f10ef: mean=1.00, good=2/2
+    7ebc8b336779: mean=1.00, good=2/2
+    8077c679fb62: mean=1.00, good=2/2
+    a67b02f4e7ce: mean=1.00, good=2/2
+    cff33035b28a: mean=1.00, good=2/2
+    f4f8079f6637: mean=1.00, good=2/2
+[Phase 2] Library update ...
+  [Library] rank=0 has 18 good samples
+  [Library] after update: SuccessLibrary(total=127, coverage=10/10, max_per_condition=64)
+[Phase 3] Re-pairing ...
+  [Re-pair] train_set: 20 samples (anchor=18, correction=2, skipped=0)
+[Phase 4] Training (100 epochs, 20 samples) ...
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1034, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 934, in main
+    train_metrics = train_one_round(
+                    ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 574, in train_one_round
+    loss.backward()
+  File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1034, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 934, in main
+[rank0]:     train_metrics = train_one_round(
+[rank0]:                     ^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 574, in train_one_round
+[rank0]:     loss.backward()
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]: RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20260408_001541-g1xdtpwt/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T00:15:41.604548Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "912b225d838164e0178805670881ee201ad81ff0"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "481117626368"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_001541-g1xdtpwt/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1601}}

wandb/run-20260408_001541-g1xdtpwt/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-08T00:15:41.614211928Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T00:15:41.614246115Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_001541-g1xdtpwt/logs/debug-core.log"}
+{"time":"2026-04-08T00:15:41.932223148Z","level":"INFO","msg":"created new stream","id":"g1xdtpwt"}
+{"time":"2026-04-08T00:15:41.932414446Z","level":"INFO","msg":"stream: started","id":"g1xdtpwt"}
+{"time":"2026-04-08T00:15:41.932510608Z","level":"INFO","msg":"sender: started","stream_id":"g1xdtpwt"}
+{"time":"2026-04-08T00:15:41.932482435Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"g1xdtpwt"}}
+{"time":"2026-04-08T00:15:41.932549715Z","level":"INFO","msg":"handler: started","stream_id":{"value":"g1xdtpwt"}}
+{"time":"2026-04-08T00:15:42.915509387Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-04-08T00:42:23.439351832Z","level":"INFO","msg":"stream: closing","id":"g1xdtpwt"}
+{"time":"2026-04-08T00:42:23.439437558Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-04-08T00:42:23.445282629Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-04-08T00:42:24.081303366Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-08T00:42:24.367520127Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"g1xdtpwt"}}
+{"time":"2026-04-08T00:42:24.367585187Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"g1xdtpwt"}}
+{"time":"2026-04-08T00:42:24.367601446Z","level":"INFO","msg":"sender: closed","stream_id":"g1xdtpwt"}
+{"time":"2026-04-08T00:42:24.371256063Z","level":"INFO","msg":"stream: closed","id":"g1xdtpwt"}

wandb/run-20260408_001541-g1xdtpwt/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 00:15:41,570 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Configure stats pid to 18968
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 00:15:41,571 INFO    MainThread:18968 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 00:15:41,573 INFO    MainThread:18968 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_001541-g1xdtpwt/logs/debug.log
+2026-04-08 00:15:41,575 INFO    MainThread:18968 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_001541-g1xdtpwt/logs/debug-internal.log
+2026-04-08 00:15:41,575 INFO    MainThread:18968 [wandb_init.py:init():621] calling init triggers
+2026-04-08 00:15:41,575 INFO    MainThread:18968 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 00:15:41,575 INFO    MainThread:18968 [wandb_init.py:init():671] starting backend
+2026-04-08 00:15:41,575 INFO    MainThread:18968 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 00:15:41,602 INFO    MainThread:18968 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 00:15:41,602 INFO    MainThread:18968 [wandb_init.py:init():688] backend started and connected
+2026-04-08 00:15:41,658 INFO    MainThread:18968 [wandb_init.py:init():783] updated telemetry
+2026-04-08 00:15:41,824 INFO    MainThread:18968 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 00:15:42,864 INFO    MainThread:18968 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 00:15:43,291 INFO    MainThread:18968 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 00:15:43,291 INFO    MainThread:18968 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 00:15:43,291 INFO    MainThread:18968 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 00:15:43,291 INFO    MainThread:18968 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 00:15:43,297 INFO    MainThread:18968 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 00:42:23,439 WARNING MsgRouterThr:18968 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_004547-x9lalqi6/files/config.yaml

@@ -0,0 +1,145 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.11.2
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.2
+            "5": 0.18.5
+            "6": 4.46.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+K:
+    value: 16
+bsv2_check_window_frac:
+    value: 0.2
+bsv2_dup_max_frames:
+    value: 0
+bsv2_expected_grip_changes:
+    value: 6
+bsv2_idm_ckpt_path:
+    value: ckpts/vidar_ckpt/idm.pt
+bsv2_mj_hi:
+    value: 1.3
+bsv2_mj_lo:
+    value: 0.6
+bsv2_pick_thr:
+    value: 0.05
+bsv2_place_thr:
+    value: 0.05
+bsv2_prompts:
+    value:
+        - red block
+        - green block
+        - blue block
+bsv2_vertical_sep_thr:
+    value: 0.03
+bsv2_x_align_thr:
+    value: 0.025
+checkpointing_steps:
+    value: 10
+ckpt_dir:
+    value: ckpts/Wan2.2-TI2V-5B
+convert_model_dtype:
+    value: true
+dataset_json:
+    value: data/rl_train/robotwin_stack_blocks_three.json
+effective_batch_size:
+    value: 16
+epochs_per_round:
+    value: 100
+frame_num:
+    value: 121
+gradient_checkpointing:
+    value: true
+hallucination_crop_top_ratio:
+    value: 0.6667
+lambda_gripper:
+    value: 1
+learning_rate:
+    value: 1e-05
+lora_alpha:
+    value: 64
+lora_rank:
+    value: 64
+lora_target_modules:
+    value: null
+max_grad_norm:
+    value: 2
+max_per_condition:
+    value: 64
+max_samples:
+    value: -1
+neg_prompt:
+    value: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+num_ode_steps:
+    value: 20
+num_rounds:
+    value: 10
+num_train_timesteps:
+    value: 1000
+offload_model:
+    value: false
+output_dir:
+    value: data/outputs/creflow_stack_blocks_three
+pt_dir:
+    value: ckpts/vidar_ckpt/merged_vidar_lora.pt
+reset_optimizer_per_round:
+    value: false
+resume_from_lora_checkpoint:
+    value: null
+reward_backend:
+    value: blocks_stack_v2
+reward_config:
+    value: blocks_stack_v2
+sample_guide_scale:
+    value: 5
+sample_shift:
+    value: 5
+seed:
+    value: 42
+size:
+    value: 640*736
+skip_reward_debug_video:
+    value: true
+task:
+    value: ti2v-5B
+use_8bit_adam:
+    value: true
+vidar_root:
+    value: ""
+w_bad:
+    value: 1
+wandb_project:
+    value: Corrective-Reflow
+wandb_run_name:
+    value: creflow_stack_blocks_three
+weight_decay:
+    value: 0.01

wandb/run-20260408_004547-x9lalqi6/files/output.log

@@ -0,0 +1,89 @@
+Building Wan2.2 TI2V model ... (DDP=True, world_size=8)
+/usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  warnings.warn(  # warn only once
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 00:47:04,252 107287 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+Traceback (most recent call last):
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 787, in __getattr__
+    return super().__getattr__(name)  # defer to nn.Module's logic
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+    raise AttributeError(
+AttributeError: 'PeftModel' object has no attribute 'enable_input_require_grads'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/tuners/lora/model.py", line 360, in __getattr__
+    return super().__getattr__(name)  # defer to nn.Module's logic
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+    raise AttributeError(
+AttributeError: 'LoraModel' object has no attribute 'enable_input_require_grads'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1035, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 770, in main
+    dit.enable_input_require_grads()
+    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 791, in __getattr__
+    return getattr(self.base_model, name)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/tuners/lora/model.py", line 364, in __getattr__
+    return getattr(self.model, name)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 173, in __getattr__
+    return super().__getattr__(name)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+    raise AttributeError(
+AttributeError: 'WanModel' object has no attribute 'enable_input_require_grads'
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 787, in __getattr__
+[rank0]:     return super().__getattr__(name)  # defer to nn.Module's logic
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+[rank0]:     raise AttributeError(
+[rank0]: AttributeError: 'PeftModel' object has no attribute 'enable_input_require_grads'
+
+[rank0]: During handling of the above exception, another exception occurred:
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/peft/tuners/lora/model.py", line 360, in __getattr__
+[rank0]:     return super().__getattr__(name)  # defer to nn.Module's logic
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+[rank0]:     raise AttributeError(
+[rank0]: AttributeError: 'LoraModel' object has no attribute 'enable_input_require_grads'
+
+[rank0]: During handling of the above exception, another exception occurred:
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1035, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 770, in main
+[rank0]:     dit.enable_input_require_grads()
+[rank0]:     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 791, in __getattr__
+[rank0]:     return getattr(self.base_model, name)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/peft/tuners/lora/model.py", line 364, in __getattr__
+[rank0]:     return getattr(self.model, name)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/diffusers/models/modeling_utils.py", line 173, in __getattr__
+[rank0]:     return super().__getattr__(name)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1964, in __getattr__
+[rank0]:     raise AttributeError(
+[rank0]: AttributeError: 'WanModel' object has no attribute 'enable_input_require_grads'

wandb/run-20260408_004547-x9lalqi6/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T00:45:47.718687Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "a8f5b170f0ef7d865af7e8f32c4aebb46efec2c1"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "481625993216"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_004547-x9lalqi6/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":80}}

wandb/run-20260408_004547-x9lalqi6/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-08T00:45:47.72832013Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T00:45:47.72833167Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004547-x9lalqi6/logs/debug-core.log"}
+{"time":"2026-04-08T00:45:47.945807814Z","level":"INFO","msg":"created new stream","id":"x9lalqi6"}
+{"time":"2026-04-08T00:45:47.945941311Z","level":"INFO","msg":"stream: started","id":"x9lalqi6"}
+{"time":"2026-04-08T00:45:47.945997568Z","level":"INFO","msg":"handler: started","stream_id":{"value":"x9lalqi6"}}
+{"time":"2026-04-08T00:45:47.946052958Z","level":"INFO","msg":"sender: started","stream_id":"x9lalqi6"}
+{"time":"2026-04-08T00:45:47.945978269Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"x9lalqi6"}}
+{"time":"2026-04-08T00:45:48.359921482Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-04-08T00:47:08.634395008Z","level":"INFO","msg":"stream: closing","id":"x9lalqi6"}
+{"time":"2026-04-08T00:47:08.634488596Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-04-08T00:47:08.639585554Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-04-08T00:47:10.458241724Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-08T00:47:10.582858773Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"x9lalqi6"}}
+{"time":"2026-04-08T00:47:10.582906095Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"x9lalqi6"}}
+{"time":"2026-04-08T00:47:10.582938514Z","level":"INFO","msg":"sender: closed","stream_id":"x9lalqi6"}
+{"time":"2026-04-08T00:47:10.586475232Z","level":"INFO","msg":"stream: closed","id":"x9lalqi6"}

wandb/run-20260408_004547-x9lalqi6/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Configure stats pid to 107287
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 00:45:47,684 INFO    MainThread:107287 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 00:45:47,686 INFO    MainThread:107287 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004547-x9lalqi6/logs/debug.log
+2026-04-08 00:45:47,688 INFO    MainThread:107287 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004547-x9lalqi6/logs/debug-internal.log
+2026-04-08 00:45:47,688 INFO    MainThread:107287 [wandb_init.py:init():621] calling init triggers
+2026-04-08 00:45:47,688 INFO    MainThread:107287 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 00:45:47,688 INFO    MainThread:107287 [wandb_init.py:init():671] starting backend
+2026-04-08 00:45:47,688 INFO    MainThread:107287 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 00:45:47,716 INFO    MainThread:107287 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 00:45:47,716 INFO    MainThread:107287 [wandb_init.py:init():688] backend started and connected
+2026-04-08 00:45:47,773 INFO    MainThread:107287 [wandb_init.py:init():783] updated telemetry
+2026-04-08 00:45:47,947 INFO    MainThread:107287 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 00:45:48,310 INFO    MainThread:107287 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 00:45:48,678 INFO    MainThread:107287 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 00:45:48,678 INFO    MainThread:107287 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 00:45:48,678 INFO    MainThread:107287 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 00:45:48,678 INFO    MainThread:107287 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 00:45:48,683 INFO    MainThread:107287 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 00:47:08,634 WARNING MsgRouterThr:107287 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_004931-ku6kqemc/files/config.yaml

@@ -0,0 +1,145 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.11.2
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.2
+            "5": 0.18.5
+            "6": 4.46.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+K:
+    value: 16
+bsv2_check_window_frac:
+    value: 0.2
+bsv2_dup_max_frames:
+    value: 0
+bsv2_expected_grip_changes:
+    value: 6
+bsv2_idm_ckpt_path:
+    value: ckpts/vidar_ckpt/idm.pt
+bsv2_mj_hi:
+    value: 1.3
+bsv2_mj_lo:
+    value: 0.6
+bsv2_pick_thr:
+    value: 0.05
+bsv2_place_thr:
+    value: 0.05
+bsv2_prompts:
+    value:
+        - red block
+        - green block
+        - blue block
+bsv2_vertical_sep_thr:
+    value: 0.03
+bsv2_x_align_thr:
+    value: 0.025
+checkpointing_steps:
+    value: 10
+ckpt_dir:
+    value: ckpts/Wan2.2-TI2V-5B
+convert_model_dtype:
+    value: true
+dataset_json:
+    value: data/rl_train/robotwin_stack_blocks_three.json
+effective_batch_size:
+    value: 16
+epochs_per_round:
+    value: 100
+frame_num:
+    value: 121
+gradient_checkpointing:
+    value: true
+hallucination_crop_top_ratio:
+    value: 0.6667
+lambda_gripper:
+    value: 1
+learning_rate:
+    value: 1e-05
+lora_alpha:
+    value: 64
+lora_rank:
+    value: 64
+lora_target_modules:
+    value: null
+max_grad_norm:
+    value: 2
+max_per_condition:
+    value: 64
+max_samples:
+    value: -1
+neg_prompt:
+    value: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+num_ode_steps:
+    value: 20
+num_rounds:
+    value: 10
+num_train_timesteps:
+    value: 1000
+offload_model:
+    value: false
+output_dir:
+    value: data/outputs/creflow_stack_blocks_three
+pt_dir:
+    value: ckpts/vidar_ckpt/merged_vidar_lora.pt
+reset_optimizer_per_round:
+    value: false
+resume_from_lora_checkpoint:
+    value: null
+reward_backend:
+    value: blocks_stack_v2
+reward_config:
+    value: blocks_stack_v2
+sample_guide_scale:
+    value: 5
+sample_shift:
+    value: 5
+seed:
+    value: 42
+size:
+    value: 640*736
+skip_reward_debug_video:
+    value: true
+task:
+    value: ti2v-5B
+use_8bit_adam:
+    value: true
+vidar_root:
+    value: ""
+w_bad:
+    value: 1
+wandb_project:
+    value: Corrective-Reflow
+wandb_run_name:
+    value: creflow_stack_blocks_three
+weight_decay:
+    value: 0.01

wandb/run-20260408_004931-ku6kqemc/files/output.log

@@ -0,0 +1,685 @@
+Building Wan2.2 TI2V model ... (DDP=True, world_size=8)
+/usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  warnings.warn(  # warn only once
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 00:50:46,165 108291 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+  Gradient checkpointing enabled on 30 DiT blocks
+Trainable parameters: 94.4 M
+  Using 8-bit AdamW (bitsandbytes)
+Dataset: 10 conditions
+Encoding 10 conditions ...
+  [1/10] 6b20973f10ef... encoded
+  [2/10] cff33035b28a... encoded
+  [3/10] 7ebc8b336779... encoded
+  [4/10] 8077c679fb62... encoded
+  [5/10] 41855438a8e0... encoded
+  [6/10] a67b02f4e7ce... encoded
+  [7/10] 4568a9603f3b... encoded
+  [8/10] 1b67584bff87... encoded
+  [9/10] f4f8079f6637... encoded
+  [10/10] 1b70247c6412... encoded
+All 10 conditions encoded.
+
+Starting Corrective Reflow | rounds=10  K=16  K_local=2  epochs/round=100  effective_bs=16  lr=1e-05  conditions=10
+
+======================================================================
+ROUND 1/10
+======================================================================
+[Phase 1] ODE rollout ...
+  [CReflow rollout] condition 0/10: K=2, id=6b20973f10ef...
+    rollout: 2 videos in 56.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.15it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:13<00:00,  9.10it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.59it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.01it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.27it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.04it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.55it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.01it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.27it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.86it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.24it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.54it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 1/10: K=2, id=cff33035b28a...
+    rollout: 2 videos in 57.1s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.62it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.25it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.35it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.43it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:11<00:00, 10.55it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.77it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 12.06it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.19it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.33it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.24it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 2/10: K=2, id=7ebc8b336779...
+    rollout: 2 videos in 57.0s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.12it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.78it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.92it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.17it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.49it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.90it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.84it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.68it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.59it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.49it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.49it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 3/10: K=2, id=8077c679fb62...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.59it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.09it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.19it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.13it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.23it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.08it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.03it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.33it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.08it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.59it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.56it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 4/10: K=2, id=41855438a8e0...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.67it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.03it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.84it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.26it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.09it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.33it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.97it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.96it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.08it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.43it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.49it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 5/10: K=2, id=a67b02f4e7ce...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.45it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.39it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.50it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.89it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.22it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.10it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.58it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.92it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.58it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.27it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.24it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 6/10: K=2, id=4568a9603f3b...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.48it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.14it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.29it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.42it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.61it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.95it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.03it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.22it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.11it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.16it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.35it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.00it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 7/10: K=2, id=1b67584bff87...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.24it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.66it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.27it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.37it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.42it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.54it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.09it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.94it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.10it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.71it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.78it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.89it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 8/10: K=2, id=f4f8079f6637...
+    rollout: 2 videos in 57.3s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.78it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.30it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.94it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.79it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.10it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.94it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.40it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.37it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 74.81it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.53it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.78it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.22it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 9/10: K=2, id=1b70247c6412...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.12it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.43it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.20it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.76it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.34it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.95it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.77it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.76it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.45it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.27it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 85.50it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.21it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] done: 20 samples, rewards = [1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 0.00, 0.00]
+  Rollout done: 20 samples, good=18, bad=2, avg_reward=0.900
+    1b67584bff87: mean=1.00, good=2/2
+    1b70247c6412: mean=0.00, good=0/2
+    41855438a8e0: mean=1.00, good=2/2
+    4568a9603f3b: mean=1.00, good=2/2
+    6b20973f10ef: mean=1.00, good=2/2
+    7ebc8b336779: mean=1.00, good=2/2
+    8077c679fb62: mean=1.00, good=2/2
+    a67b02f4e7ce: mean=1.00, good=2/2
+    cff33035b28a: mean=1.00, good=2/2
+    f4f8079f6637: mean=1.00, good=2/2
+[Phase 2] Library update ...
+  [Library] rank=0 has 18 good samples
+  [Library] after update: SuccessLibrary(total=127, coverage=10/10, max_per_condition=64)
+[Phase 3] Re-pairing ...
+  [Re-pair] train_set: 20 samples (anchor=18, correction=2, skipped=0)
+[Phase 4] Training (100 epochs, 20 samples) ...
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1037, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 937, in main
+    train_metrics = train_one_round(
+                    ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 577, in train_one_round
+    loss.backward()
+  File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 1158, in unpack_hook
+    frame.check_recomputed_tensors_match(gid)
+  File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 911, in check_recomputed_tensors_match
+    raise CheckpointError(
+torch.utils.checkpoint.CheckpointError: torch.utils.checkpoint: Recomputed values for the following tensors have different metadata than during the forward pass.
+tensor at position 6:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 7:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 8:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 9:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 10:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 11:
+saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 12:
+saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 13:
+saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 14:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 15:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 16:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 17:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 18:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 19:
+saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 20:
+saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 21:
+saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 22:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 23:
+saved metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 24:
+saved metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 25:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 26:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 27:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 28:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 29:
+saved metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+tensor at position 30:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+tensor at position 31:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 32:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 33:
+saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 34:
+saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 35:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 36:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+tensor at position 37:
+saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+tensor at position 38:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+tensor at position 39:
+saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 40:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 41:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 42:
+saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 43:
+saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 44:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 45:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 46:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 47:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 48:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 49:
+saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 50:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 51:
+saved metadata: {'shape': torch.Size([512, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 52:
+saved metadata: {'shape': torch.Size([512, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 53:
+saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 54:
+saved metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 55:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 56:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 57:
+saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 58:
+saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 59:
+saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 60:
+saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([512, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 61:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 62:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([512, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 63:
+saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 512, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 64:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 512, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 65:
+saved metadata: {'shape': torch.Size([3072, 14336]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 512, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 66:
+saved metadata: {'shape': torch.Size([1, 14260, 14336]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([1, 512, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+tensor at position 67:
+saved metadata: {'shape': torch.Size([14336, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+tensor at position 68:
+saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+recomputed metadata: {'shape': torch.Size([512, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+.
+
+Tip: To see a more detailed error message, either pass `debug=True` to
+`torch.utils.checkpoint.checkpoint(...)` or wrap the code block
+with `with torch.utils.checkpoint.set_checkpoint_debug_enabled(True):` to
+enable checkpoint‑debug mode globally.
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1037, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 937, in main
+[rank0]:     train_metrics = train_one_round(
+[rank0]:                     ^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 577, in train_one_round
+[rank0]:     loss.backward()
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 1158, in unpack_hook
+[rank0]:     frame.check_recomputed_tensors_match(gid)
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 911, in check_recomputed_tensors_match
+[rank0]:     raise CheckpointError(
+[rank0]: torch.utils.checkpoint.CheckpointError: torch.utils.checkpoint: Recomputed values for the following tensors have different metadata than during the forward pass.
+[rank0]: tensor at position 6:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 7:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 8:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 9:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 10:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 11:
+[rank0]: saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 12:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 13:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 14:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 15:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 16:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 17:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 18:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 19:
+[rank0]: saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 20:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 21:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 22:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 23:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 24:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 25:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 26:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 27:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 28:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 29:
+[rank0]: saved metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 30:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 1, 64]), 'dtype': torch.complex128, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 31:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 32:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 33:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 34:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 35:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 36:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 37:
+[rank0]: saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 38:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 39:
+[rank0]: saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 40:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 41:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 42:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 43:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 44:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 45:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 46:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 47:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 48:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 49:
+[rank0]: saved metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 50:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 51:
+[rank0]: saved metadata: {'shape': torch.Size([512, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 52:
+[rank0]: saved metadata: {'shape': torch.Size([512, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 53:
+[rank0]: saved metadata: {'shape': torch.Size([14260, 24, 128]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 54:
+[rank0]: saved metadata: {'shape': torch.Size([24, 14260]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([14260, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 55:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 56:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 57:
+[rank0]: saved metadata: {'shape': torch.Size([2]), 'dtype': torch.int64, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 58:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 59:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 60:
+[rank0]: saved metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([512, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 61:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([64, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 62:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([512, 64]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 63:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 512, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 64:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 512, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 65:
+[rank0]: saved metadata: {'shape': torch.Size([3072, 14336]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 512, 1]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 66:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 14336]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([1, 512, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 67:
+[rank0]: saved metadata: {'shape': torch.Size([14336, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: tensor at position 68:
+[rank0]: saved metadata: {'shape': torch.Size([1, 14260, 3072]), 'dtype': torch.float32, 'device': device(type='cuda', index=0)}
+[rank0]: recomputed metadata: {'shape': torch.Size([512, 3072]), 'dtype': torch.bfloat16, 'device': device(type='cuda', index=0)}
+[rank0]: .
+
+[rank0]: Tip: To see a more detailed error message, either pass `debug=True` to
+[rank0]: `torch.utils.checkpoint.checkpoint(...)` or wrap the code block
+[rank0]: with `with torch.utils.checkpoint.set_checkpoint_debug_enabled(True):` to
+[rank0]: enable checkpoint‑debug mode globally.

wandb/run-20260408_004931-ku6kqemc/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T00:49:31.229785Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "a8f5b170f0ef7d865af7e8f32c4aebb46efec2c1"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "481703223296"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_004931-ku6kqemc/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1584}}

wandb/run-20260408_004931-ku6kqemc/logs/debug-internal.log

@@ -0,0 +1,19 @@
+{"time":"2026-04-08T00:49:31.238719294Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T00:49:31.23873382Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004931-ku6kqemc/logs/debug-core.log"}
+{"time":"2026-04-08T00:49:31.455165146Z","level":"INFO","msg":"created new stream","id":"ku6kqemc"}
+{"time":"2026-04-08T00:49:31.455371786Z","level":"INFO","msg":"stream: started","id":"ku6kqemc"}
+{"time":"2026-04-08T00:49:31.455479963Z","level":"INFO","msg":"sender: started","stream_id":"ku6kqemc"}
+{"time":"2026-04-08T00:49:31.455431707Z","level":"INFO","msg":"handler: started","stream_id":{"value":"ku6kqemc"}}
+{"time":"2026-04-08T00:49:31.455410767Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"ku6kqemc"}}
+{"time":"2026-04-08T00:49:31.831542842Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-04-08T01:00:32.624028988Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-04-08T01:01:05.122882271Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-04-08T01:09:32.404506925Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/vincentni/Corrective-Reflow/ku6kqemc/file_stream\": dial tcp 35.186.228.49:443: connect: connection timed out"}
+{"time":"2026-04-08T01:15:56.227072593Z","level":"INFO","msg":"stream: closing","id":"ku6kqemc"}
+{"time":"2026-04-08T01:15:56.227162615Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-04-08T01:15:56.232220483Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-04-08T01:15:56.791766357Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-08T01:15:56.926286921Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"ku6kqemc"}}
+{"time":"2026-04-08T01:15:56.926354182Z","level":"INFO","msg":"sender: closed","stream_id":"ku6kqemc"}
+{"time":"2026-04-08T01:15:56.926329752Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"ku6kqemc"}}
+{"time":"2026-04-08T01:15:56.930384735Z","level":"INFO","msg":"stream: closed","id":"ku6kqemc"}

wandb/run-20260408_004931-ku6kqemc/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 00:49:31,195 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 00:49:31,195 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Configure stats pid to 108291
+2026-04-08 00:49:31,195 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 00:49:31,196 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 00:49:31,196 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 00:49:31,196 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 00:49:31,196 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 00:49:31,196 INFO    MainThread:108291 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 00:49:31,198 INFO    MainThread:108291 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004931-ku6kqemc/logs/debug.log
+2026-04-08 00:49:31,199 INFO    MainThread:108291 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_004931-ku6kqemc/logs/debug-internal.log
+2026-04-08 00:49:31,200 INFO    MainThread:108291 [wandb_init.py:init():621] calling init triggers
+2026-04-08 00:49:31,200 INFO    MainThread:108291 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 00:49:31,200 INFO    MainThread:108291 [wandb_init.py:init():671] starting backend
+2026-04-08 00:49:31,200 INFO    MainThread:108291 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 00:49:31,227 INFO    MainThread:108291 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 00:49:31,227 INFO    MainThread:108291 [wandb_init.py:init():688] backend started and connected
+2026-04-08 00:49:31,285 INFO    MainThread:108291 [wandb_init.py:init():783] updated telemetry
+2026-04-08 00:49:31,449 INFO    MainThread:108291 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 00:49:31,780 INFO    MainThread:108291 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 00:49:32,146 INFO    MainThread:108291 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 00:49:32,146 INFO    MainThread:108291 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 00:49:32,146 INFO    MainThread:108291 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 00:49:32,146 INFO    MainThread:108291 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 00:49:32,151 INFO    MainThread:108291 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 01:15:56,227 WARNING MsgRouterThr:108291 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_012032-idkfwxb0/files/config.yaml

@@ -0,0 +1,145 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.11.2
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.2
+            "5": 0.18.5
+            "6": 4.46.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+K:
+    value: 16
+bsv2_check_window_frac:
+    value: 0.2
+bsv2_dup_max_frames:
+    value: 0
+bsv2_expected_grip_changes:
+    value: 6
+bsv2_idm_ckpt_path:
+    value: ckpts/vidar_ckpt/idm.pt
+bsv2_mj_hi:
+    value: 1.3
+bsv2_mj_lo:
+    value: 0.6
+bsv2_pick_thr:
+    value: 0.05
+bsv2_place_thr:
+    value: 0.05
+bsv2_prompts:
+    value:
+        - red block
+        - green block
+        - blue block
+bsv2_vertical_sep_thr:
+    value: 0.03
+bsv2_x_align_thr:
+    value: 0.025
+checkpointing_steps:
+    value: 10
+ckpt_dir:
+    value: ckpts/Wan2.2-TI2V-5B
+convert_model_dtype:
+    value: true
+dataset_json:
+    value: data/rl_train/robotwin_stack_blocks_three.json
+effective_batch_size:
+    value: 16
+epochs_per_round:
+    value: 100
+frame_num:
+    value: 121
+gradient_checkpointing:
+    value: true
+hallucination_crop_top_ratio:
+    value: 0.6667
+lambda_gripper:
+    value: 1
+learning_rate:
+    value: 1e-05
+lora_alpha:
+    value: 64
+lora_rank:
+    value: 64
+lora_target_modules:
+    value: null
+max_grad_norm:
+    value: 2
+max_per_condition:
+    value: 64
+max_samples:
+    value: -1
+neg_prompt:
+    value: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+num_ode_steps:
+    value: 20
+num_rounds:
+    value: 10
+num_train_timesteps:
+    value: 1000
+offload_model:
+    value: false
+output_dir:
+    value: data/outputs/creflow_stack_blocks_three
+pt_dir:
+    value: ckpts/vidar_ckpt/merged_vidar_lora.pt
+reset_optimizer_per_round:
+    value: false
+resume_from_lora_checkpoint:
+    value: null
+reward_backend:
+    value: blocks_stack_v2
+reward_config:
+    value: blocks_stack_v2
+sample_guide_scale:
+    value: 5
+sample_shift:
+    value: 5
+seed:
+    value: 42
+size:
+    value: 640*736
+skip_reward_debug_video:
+    value: true
+task:
+    value: ti2v-5B
+use_8bit_adam:
+    value: true
+vidar_root:
+    value: ""
+w_bad:
+    value: 1
+wandb_project:
+    value: Corrective-Reflow
+wandb_run_name:
+    value: creflow_stack_blocks_three
+weight_decay:
+    value: 0.01

wandb/run-20260408_012032-idkfwxb0/files/output.log

@@ -0,0 +1,133 @@
+Building Wan2.2 TI2V model ... (DDP=True, world_size=8)
+/usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  warnings.warn(  # warn only once
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 01:21:50,257 195712 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+  Gradient checkpointing enabled on 30 DiT blocks
+Trainable parameters: 94.4 M
+  Using 8-bit AdamW (bitsandbytes)
+Dataset: 10 conditions
+Encoding 10 conditions ...
+  [1/10] 6b20973f10ef... encoded
+  [2/10] cff33035b28a... encoded
+  [3/10] 7ebc8b336779... encoded
+  [4/10] 8077c679fb62... encoded
+  [5/10] 41855438a8e0... encoded
+  [6/10] a67b02f4e7ce... encoded
+  [7/10] 4568a9603f3b... encoded
+  [8/10] 1b67584bff87... encoded
+  [9/10] f4f8079f6637... encoded
+  [10/10] 1b70247c6412... encoded
+All 10 conditions encoded.
+
+Starting Corrective Reflow | rounds=10  K=16  K_local=2  epochs/round=100  effective_bs=16  lr=1e-05  conditions=10
+
+======================================================================
+ROUND 1/10
+======================================================================
+[Phase 1] ODE rollout ...
+  [CReflow rollout] condition 0/10: K=2, id=6b20973f10ef...
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1038, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 870, in main
+    rollouts = creflow_rollout(
+               ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 267, in creflow_rollout
+    x0_preds = _ode_rollout_batch(
+               ^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 139, in _ode_rollout_batch
+    outputs = dit(x_list, t=ts_batch, context=context_list, seq_len=seq_len)
+              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 812, in forward
+    return self.get_base_model()(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 498, in forward
+    x = block(x, **kwargs)
+        ^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 130, in _ckpt_fwd
+    return torch_ckpt.checkpoint(orig_fwd, *args, use_reentrant=True, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_compile.py", line 53, in inner
+    return disable_fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 486, in checkpoint
+    raise ValueError(
+ValueError: Unexpected keyword arguments: e,seq_lens,grid_sizes,freqs,context,context_lens
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1038, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 870, in main
+[rank0]:     rollouts = creflow_rollout(
+[rank0]:                ^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 267, in creflow_rollout
+[rank0]:     x0_preds = _ode_rollout_batch(
+[rank0]:                ^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/creflow/rollout.py", line 139, in _ode_rollout_batch
+[rank0]:     outputs = dit(x_list, t=ts_batch, context=context_list, seq_len=seq_len)
+[rank0]:               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/home/tiger/.local/lib/python3.11/site-packages/peft/peft_model.py", line 812, in forward
+[rank0]:     return self.get_base_model()(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/models/wan/modules/model.py", line 498, in forward
+[rank0]:     x = block(x, **kwargs)
+[rank0]:         ^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1786, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 130, in _ckpt_fwd
+[rank0]:     return torch_ckpt.checkpoint(orig_fwd, *args, use_reentrant=True, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/_compile.py", line 53, in inner
+[rank0]:     return disable_fn(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/_dynamo/eval_frame.py", line 1044, in _fn
+[rank0]:     return fn(*args, **kwargs)
+[rank0]:            ^^^^^^^^^^^^^^^^^^^
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/utils/checkpoint.py", line 486, in checkpoint
+[rank0]:     raise ValueError(
+[rank0]: ValueError: Unexpected keyword arguments: e,seq_lens,grid_sizes,freqs,context,context_lens

wandb/run-20260408_012032-idkfwxb0/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T01:20:32.358609Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "a8f5b170f0ef7d865af7e8f32c4aebb46efec2c1"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "482171138048"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_012032-idkfwxb0/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":84}}

wandb/run-20260408_012032-idkfwxb0/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-08T01:20:32.367150627Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T01:20:32.367162484Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_012032-idkfwxb0/logs/debug-core.log"}
+{"time":"2026-04-08T01:20:32.581603896Z","level":"INFO","msg":"created new stream","id":"idkfwxb0"}
+{"time":"2026-04-08T01:20:32.581769665Z","level":"INFO","msg":"stream: started","id":"idkfwxb0"}
+{"time":"2026-04-08T01:20:32.5818252Z","level":"INFO","msg":"sender: started","stream_id":"idkfwxb0"}
+{"time":"2026-04-08T01:20:32.581798295Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"idkfwxb0"}}
+{"time":"2026-04-08T01:20:32.581839505Z","level":"INFO","msg":"handler: started","stream_id":{"value":"idkfwxb0"}}
+{"time":"2026-04-08T01:20:32.990097512Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-04-08T01:21:57.340247846Z","level":"INFO","msg":"stream: closing","id":"idkfwxb0"}
+{"time":"2026-04-08T01:21:57.340446284Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-04-08T01:21:57.346621494Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-04-08T01:21:58.004298171Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-08T01:21:58.157765962Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"idkfwxb0"}}
+{"time":"2026-04-08T01:21:58.157800232Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"idkfwxb0"}}
+{"time":"2026-04-08T01:21:58.157836905Z","level":"INFO","msg":"sender: closed","stream_id":"idkfwxb0"}
+{"time":"2026-04-08T01:21:58.16571198Z","level":"INFO","msg":"stream: closed","id":"idkfwxb0"}

wandb/run-20260408_012032-idkfwxb0/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Configure stats pid to 195712
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 01:20:32,325 INFO    MainThread:195712 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 01:20:32,327 INFO    MainThread:195712 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_012032-idkfwxb0/logs/debug.log
+2026-04-08 01:20:32,329 INFO    MainThread:195712 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_012032-idkfwxb0/logs/debug-internal.log
+2026-04-08 01:20:32,329 INFO    MainThread:195712 [wandb_init.py:init():621] calling init triggers
+2026-04-08 01:20:32,329 INFO    MainThread:195712 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 01:20:32,329 INFO    MainThread:195712 [wandb_init.py:init():671] starting backend
+2026-04-08 01:20:32,329 INFO    MainThread:195712 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 01:20:32,356 INFO    MainThread:195712 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 01:20:32,356 INFO    MainThread:195712 [wandb_init.py:init():688] backend started and connected
+2026-04-08 01:20:32,412 INFO    MainThread:195712 [wandb_init.py:init():783] updated telemetry
+2026-04-08 01:20:32,575 INFO    MainThread:195712 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 01:20:32,942 INFO    MainThread:195712 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 01:20:33,301 INFO    MainThread:195712 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 01:20:33,301 INFO    MainThread:195712 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 01:20:33,301 INFO    MainThread:195712 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 01:20:33,301 INFO    MainThread:195712 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 01:20:33,306 INFO    MainThread:195712 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 01:21:57,340 WARNING MsgRouterThr:195712 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_014250-d3p1n5w7/files/config.yaml

@@ -0,0 +1,145 @@
+_wandb:
+    value:
+        cli_version: 0.18.5
+        m: []
+        python_version: 3.11.2
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 55
+                - 63
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 13
+                - 16
+                - 23
+                - 55
+            "4": 3.11.2
+            "5": 0.18.5
+            "6": 4.46.1
+            "8":
+                - 5
+            "12": 0.18.5
+            "13": linux-x86_64
+K:
+    value: 16
+bsv2_check_window_frac:
+    value: 0.2
+bsv2_dup_max_frames:
+    value: 0
+bsv2_expected_grip_changes:
+    value: 6
+bsv2_idm_ckpt_path:
+    value: ckpts/vidar_ckpt/idm.pt
+bsv2_mj_hi:
+    value: 1.3
+bsv2_mj_lo:
+    value: 0.6
+bsv2_pick_thr:
+    value: 0.05
+bsv2_place_thr:
+    value: 0.05
+bsv2_prompts:
+    value:
+        - red block
+        - green block
+        - blue block
+bsv2_vertical_sep_thr:
+    value: 0.03
+bsv2_x_align_thr:
+    value: 0.025
+checkpointing_steps:
+    value: 10
+ckpt_dir:
+    value: ckpts/Wan2.2-TI2V-5B
+convert_model_dtype:
+    value: true
+dataset_json:
+    value: data/rl_train/robotwin_stack_blocks_three.json
+effective_batch_size:
+    value: 16
+epochs_per_round:
+    value: 100
+frame_num:
+    value: 121
+gradient_checkpointing:
+    value: true
+hallucination_crop_top_ratio:
+    value: 0.6667
+lambda_gripper:
+    value: 1
+learning_rate:
+    value: 1e-05
+lora_alpha:
+    value: 64
+lora_rank:
+    value: 64
+lora_target_modules:
+    value: null
+max_grad_norm:
+    value: 2
+max_per_condition:
+    value: 64
+max_samples:
+    value: -1
+neg_prompt:
+    value: 色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走
+num_ode_steps:
+    value: 20
+num_rounds:
+    value: 10
+num_train_timesteps:
+    value: 1000
+offload_model:
+    value: false
+output_dir:
+    value: data/outputs/creflow_stack_blocks_three
+pt_dir:
+    value: ckpts/vidar_ckpt/merged_vidar_lora.pt
+reset_optimizer_per_round:
+    value: false
+resume_from_lora_checkpoint:
+    value: null
+reward_backend:
+    value: blocks_stack_v2
+reward_config:
+    value: blocks_stack_v2
+sample_guide_scale:
+    value: 5
+sample_shift:
+    value: 5
+seed:
+    value: 42
+size:
+    value: 640*736
+skip_reward_debug_video:
+    value: true
+task:
+    value: ti2v-5B
+use_8bit_adam:
+    value: true
+vidar_root:
+    value: ""
+w_bad:
+    value: 1
+wandb_project:
+    value: Corrective-Reflow
+wandb_run_name:
+    value: creflow_stack_blocks_three
+weight_decay:
+    value: 0.01

wandb/run-20260408_014250-d3p1n5w7/files/output.log

@@ -0,0 +1,287 @@
+Building Wan2.2 TI2V model ... (DDP=True, world_size=8)
+/usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4876: UserWarning: barrier(): using the device under current context. You can specify `device_id` in `init_process_group` to mute this warning.
+  warnings.warn(  # warn only once
+  Moving T5 encoder to GPU permanently (offload_model=false) ...
+Building reward scorer ...
+  Loading IDM from ckpts/vidar_ckpt/idm.pt on cuda:0 ...
+  IDM loaded.
+  Loading SAM3 VideoPredictor (blocks_stack_v2) on cuda:0 ...
+INFO 2026-04-08 01:44:06,994 198734 sam3_video_base.py: 125: setting max_num_objects=10000 and num_obj_for_compile=16
+  SAM3 VideoPredictor (blocks_stack_v2) loaded.
+Applying LoRA (single adapter) for CReflow ...
+  LoRA injected: rank=64, alpha=64, target_modules=['self_attn.q', 'self_attn.k', 'self_attn.v', 'self_attn.o', 'cross_attn.q', 'cross_attn.k', 'cross_attn.v', 'cross_attn.o']
+  Trainable: 94.4 M / 5094.2 M total (1.85%)
+  Gradient checkpointing enabled on 30 DiT blocks
+Trainable parameters: 94.4 M
+  Using 8-bit AdamW (bitsandbytes)
+Dataset: 10 conditions
+Encoding 10 conditions ...
+  [1/10] 6b20973f10ef... encoded
+  [2/10] cff33035b28a... encoded
+  [3/10] 7ebc8b336779... encoded
+  [4/10] 8077c679fb62... encoded
+  [5/10] 41855438a8e0... encoded
+  [6/10] a67b02f4e7ce... encoded
+  [7/10] 4568a9603f3b... encoded
+  [8/10] 1b67584bff87... encoded
+  [9/10] f4f8079f6637... encoded
+  [10/10] 1b70247c6412... encoded
+All 10 conditions encoded.
+
+Starting Corrective Reflow | rounds=10  K=16  K_local=2  epochs/round=100  effective_bs=16  lr=1e-05  conditions=10
+
+======================================================================
+ROUND 1/10
+======================================================================
+[Phase 1] ODE rollout ...
+  [CReflow rollout] condition 0/10: K=2, id=6b20973f10ef...
+    rollout: 2 videos in 56.8s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.44it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:11<00:00, 10.30it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.75it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.78it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.49it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.74it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 72.64it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.57it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.74it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.79it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.37it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.02it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 1/10: K=2, id=cff33035b28a...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.04it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.14it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.48it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.66it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.72it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.64it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.93it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.33it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.69it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.32it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.65it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.69it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 2/10: K=2, id=7ebc8b336779...
+    rollout: 2 videos in 57.1s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.80it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.71it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.51it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.17it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.31it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.71it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 77.59it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.19it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.99it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.72it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.19it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.99it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 3/10: K=2, id=8077c679fb62...
+    rollout: 2 videos in 57.3s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 75.88it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.54it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.33it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 14.03it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.71it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.97it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.94it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.80it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.48it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.06it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.86it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 4/10: K=2, id=41855438a8e0...
+    rollout: 2 videos in 57.6s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.59it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.88it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.29it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.15it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.15it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.88it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.97it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.74it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.05it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.30it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.51it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.83it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 5/10: K=2, id=a67b02f4e7ce...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.02it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.57it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.85it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.65it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.17it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.58it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.90it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.12it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.12it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.62it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.90it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.09it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 6/10: K=2, id=4568a9603f3b...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.94it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.46it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.68it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.73it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.83it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.45it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.64it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.76it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.14it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.60it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.70it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.80it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 7/10: K=2, id=1b67584bff87...
+    rollout: 2 videos in 57.0s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 75.90it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 11.75it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.26it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.18it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.63it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.95it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 79.39it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.61it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.79it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.57it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.72it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.06it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 8/10: K=2, id=f4f8079f6637...
+    rollout: 2 videos in 57.5s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.04it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 12.04it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 82.37it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.40it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.08it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.58it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.47it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.20it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.48it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.30it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 78.85it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.52it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] condition 9/10: K=2, id=1b70247c6412...
+    rollout: 2 videos in 56.8s
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.06it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:08<00:00, 13.54it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 83.74it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.27it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 81.74it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.43it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.00it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:10<00:00, 12.07it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 84.11it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 12.98it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+frame loading (image folder) [rank=0]: 100%|██████████| 121/121 [00:01<00:00, 80.99it/s]
+propagate_in_video: 100%|██████████| 121/121 [00:09<00:00, 13.26it/s]
+propagate_in_video: 0it [00:00, ?it/s]
+  [CReflow rollout] done: 20 samples, rewards = [1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 1.00, 0.00, 0.00]
+  Rollout done: 20 samples, good=18, bad=2, avg_reward=0.900
+    1b67584bff87: mean=1.00, good=2/2
+    1b70247c6412: mean=0.00, good=0/2
+    41855438a8e0: mean=1.00, good=2/2
+    4568a9603f3b: mean=1.00, good=2/2
+    6b20973f10ef: mean=1.00, good=2/2
+    7ebc8b336779: mean=1.00, good=2/2
+    8077c679fb62: mean=1.00, good=2/2
+    a67b02f4e7ce: mean=1.00, good=2/2
+    cff33035b28a: mean=1.00, good=2/2
+    f4f8079f6637: mean=1.00, good=2/2
+[Phase 2] Library update ...
+  [Library] rank=0 has 18 good samples
+  [Library] after update: SuccessLibrary(total=127, coverage=10/10, max_per_condition=64)
+[Phase 3] Re-pairing ...
+  [Re-pair] train_set: 20 samples (anchor=18, correction=2, skipped=0)
+[Phase 4] Training (100 epochs, 20 samples) ...
+    [DEBUG] v_pred requires_grad=False, grad_fn=False
+Traceback (most recent call last):
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1039, in <module>
+    main()
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 939, in main
+    train_metrics = train_one_round(
+                    ^^^^^^^^^^^^^^^^
+  File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 579, in train_one_round
+    loss.backward()
+  File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+    _engine_run_backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+    return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 1039, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 939, in main
+[rank0]:     train_metrics = train_one_round(
+[rank0]:                     ^^^^^^^^^^^^^^^^
+[rank0]:   File "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py", line 579, in train_one_round
+[rank0]:     loss.backward()
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 625, in backward
+[rank0]:     torch.autograd.backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 354, in backward
+[rank0]:     _engine_run_backward(
+[rank0]:   File "/usr/local/lib/python3.11/dist-packages/torch/autograd/graph.py", line 841, in _engine_run_backward
+[rank0]:     return Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+[rank0]: RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

wandb/run-20260408_014250-d3p1n5w7/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T01:42:50.836955Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "a8f5b170f0ef7d865af7e8f32c4aebb46efec2c1"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-104-180",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "482529443840"
+    }
+  },
+  "memory":  {
+    "total":  "1977660338176"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}

wandb/run-20260408_014250-d3p1n5w7/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":1586}}

wandb/run-20260408_014250-d3p1n5w7/logs/debug-internal.log

@@ -0,0 +1,18 @@
+{"time":"2026-04-08T01:42:50.846912727Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T01:42:50.846924973Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_014250-d3p1n5w7/logs/debug-core.log"}
+{"time":"2026-04-08T01:42:51.065325741Z","level":"INFO","msg":"created new stream","id":"d3p1n5w7"}
+{"time":"2026-04-08T01:42:51.06552384Z","level":"INFO","msg":"stream: started","id":"d3p1n5w7"}
+{"time":"2026-04-08T01:42:51.065611464Z","level":"INFO","msg":"sender: started","stream_id":"d3p1n5w7"}
+{"time":"2026-04-08T01:42:51.06558469Z","level":"INFO","msg":"handler: started","stream_id":{"value":"d3p1n5w7"}}
+{"time":"2026-04-08T01:42:51.065564877Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"d3p1n5w7"}}
+{"time":"2026-04-08T01:42:51.47008492Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-04-08T02:00:36.824476956Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-04-08T02:01:09.269479394Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-04-08T02:09:17.196524875Z","level":"INFO","msg":"stream: closing","id":"d3p1n5w7"}
+{"time":"2026-04-08T02:09:17.196574831Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-04-08T02:09:17.201147257Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-04-08T02:09:18.01646151Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-08T02:09:18.177322397Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"d3p1n5w7"}}
+{"time":"2026-04-08T02:09:18.177361768Z","level":"INFO","msg":"sender: closed","stream_id":"d3p1n5w7"}
+{"time":"2026-04-08T02:09:18.177361574Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"d3p1n5w7"}}
+{"time":"2026-04-08T02:09:18.183256398Z","level":"INFO","msg":"stream: closed","id":"d3p1n5w7"}

wandb/run-20260408_014250-d3p1n5w7/logs/debug.log

@@ -0,0 +1,26 @@
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Current SDK version is 0.18.5
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Configure stats pid to 198734
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Loading settings from /home/tiger/.config/wandb/settings
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Loading settings from /mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/wandb/settings
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'disabled': 'false', 'project': 'Corrective-Reflow', 'api_key': '***REDACTED***', 'mode': 'online'}
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'fastvideo/train_creflow_wan_2_2_ti2v.py', 'program_abspath': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py', 'program': '/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py'}
+2026-04-08 01:42:50,803 INFO    MainThread:198734 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-04-08 01:42:50,805 INFO    MainThread:198734 [wandb_init.py:_log_setup():534] Logging user logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_014250-d3p1n5w7/logs/debug.log
+2026-04-08 01:42:50,807 INFO    MainThread:198734 [wandb_init.py:_log_setup():535] Logging internal logs to data/outputs/creflow_stack_blocks_three/wandb/run-20260408_014250-d3p1n5w7/logs/debug-internal.log
+2026-04-08 01:42:50,807 INFO    MainThread:198734 [wandb_init.py:init():621] calling init triggers
+2026-04-08 01:42:50,807 INFO    MainThread:198734 [wandb_init.py:init():628] wandb.init called with sweep_config: {}
+config: {'vidar_root': '', 'task': 'ti2v-5B', 'size': '640*736', 'frame_num': 121, 'ckpt_dir': 'ckpts/Wan2.2-TI2V-5B', 'pt_dir': 'ckpts/vidar_ckpt/merged_vidar_lora.pt', 'convert_model_dtype': True, 'offload_model': False, 'dataset_json': 'data/rl_train/robotwin_stack_blocks_three.json', 'max_samples': -1, 'output_dir': 'data/outputs/creflow_stack_blocks_three', 'num_ode_steps': 20, 'sample_shift': 5.0, 'sample_guide_scale': 5.0, 'neg_prompt': '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走', 'seed': 42, 'K': 16, 'num_rounds': 10, 'epochs_per_round': 100, 'effective_batch_size': 16, 'w_bad': 1.0, 'max_per_condition': 64, 'lambda_gripper': 1.0, 'reset_optimizer_per_round': False, 'num_train_timesteps': 1000, 'learning_rate': 1e-05, 'weight_decay': 0.01, 'max_grad_norm': 2.0, 'checkpointing_steps': 10, 'gradient_checkpointing': True, 'use_8bit_adam': True, 'lora_rank': 64, 'lora_alpha': 64, 'lora_target_modules': None, 'resume_from_lora_checkpoint': None, 'reward_config': 'blocks_stack_v2', 'reward_backend': 'blocks_stack_v2', 'skip_reward_debug_video': True, 'wandb_project': 'Corrective-Reflow', 'wandb_run_name': 'creflow_stack_blocks_three', 'hallucination_crop_top_ratio': 0.6667, 'bsv2_prompts': ['red block', 'green block', 'blue block'], 'bsv2_pick_thr': 0.05, 'bsv2_place_thr': 0.05, 'bsv2_vertical_sep_thr': 0.03, 'bsv2_x_align_thr': 0.025, 'bsv2_check_window_frac': 0.2, 'bsv2_dup_max_frames': 0, 'bsv2_expected_grip_changes': 6, 'bsv2_mj_lo': 0.6, 'bsv2_mj_hi': 1.3, 'bsv2_idm_ckpt_path': 'ckpts/vidar_ckpt/idm.pt'}
+2026-04-08 01:42:50,807 INFO    MainThread:198734 [wandb_init.py:init():671] starting backend
+2026-04-08 01:42:50,807 INFO    MainThread:198734 [wandb_init.py:init():675] sending inform_init request
+2026-04-08 01:42:50,834 INFO    MainThread:198734 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-04-08 01:42:50,835 INFO    MainThread:198734 [wandb_init.py:init():688] backend started and connected
+2026-04-08 01:42:50,889 INFO    MainThread:198734 [wandb_init.py:init():783] updated telemetry
+2026-04-08 01:42:51,055 INFO    MainThread:198734 [wandb_init.py:init():816] communicating run to backend with 90.0 second timeout
+2026-04-08 01:42:51,420 INFO    MainThread:198734 [wandb_init.py:init():867] starting run threads in backend
+2026-04-08 01:42:51,784 INFO    MainThread:198734 [wandb_run.py:_console_start():2463] atexit reg
+2026-04-08 01:42:51,785 INFO    MainThread:198734 [wandb_run.py:_redirect():2311] redirect: wrap_raw
+2026-04-08 01:42:51,785 INFO    MainThread:198734 [wandb_run.py:_redirect():2376] Wrapping output streams.
+2026-04-08 01:42:51,785 INFO    MainThread:198734 [wandb_run.py:_redirect():2401] Redirects installed.
+2026-04-08 01:42:51,790 INFO    MainThread:198734 [wandb_init.py:init():911] run started, returning control to user process
+2026-04-08 02:09:17,196 WARNING MsgRouterThr:198734 [router.py:message_loop():77] message_loop has been closed

wandb/run-20260408_025012-pf74wm1f/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2026-04-08T02:50:12.360136031Z","level":"INFO","msg":"using version","core version":"0.18.5"}
+{"time":"2026-04-08T02:50:12.360147Z","level":"INFO","msg":"created symlink","path":"data/outputs/creflow_stack_blocks_three/wandb/run-20260408_025012-pf74wm1f/logs/debug-core.log"}
+{"time":"2026-04-08T02:50:12.578898735Z","level":"INFO","msg":"created new stream","id":"pf74wm1f"}
+{"time":"2026-04-08T02:50:12.579095021Z","level":"INFO","msg":"stream: started","id":"pf74wm1f"}
+{"time":"2026-04-08T02:50:12.5791654Z","level":"INFO","msg":"sender: started","stream_id":"pf74wm1f"}
+{"time":"2026-04-08T02:50:12.579140373Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"pf74wm1f"}}
+{"time":"2026-04-08T02:50:12.579178443Z","level":"INFO","msg":"handler: started","stream_id":{"value":"pf74wm1f"}}
+{"time":"2026-04-08T02:50:12.901012044Z","level":"INFO","msg":"Starting system monitor"}

wandb/run-20260408_164919-x63spn6n/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-5.15.120.bsk.6-amd64-x86_64-with-glibc2.36",
+  "python":  "3.11.2",
+  "startedAt":  "2026-04-08T16:49:19.417199Z",
+  "args":  [
+    "--task",
+    "ti2v-5B",
+    "--size",
+    "640*736",
+    "--frame_num",
+    "121",
+    "--ckpt_dir",
+    "ckpts/Wan2.2-TI2V-5B",
+    "--pt_dir",
+    "ckpts/vidar_ckpt/merged_vidar_lora.pt",
+    "--dataset_json",
+    "data/rl_train/robotwin_stack_blocks_three.json",
+    "--output_dir",
+    "data/outputs/creflow_stack_blocks_three",
+    "--num_ode_steps",
+    "20",
+    "--sample_shift",
+    "5.0",
+    "--sample_guide_scale",
+    "5.0",
+    "--K",
+    "16",
+    "--num_rounds",
+    "10",
+    "--epochs_per_round",
+    "100",
+    "--max_per_condition",
+    "64",
+    "--effective_batch_size",
+    "16",
+    "--w_bad",
+    "1.0",
+    "--lambda_gripper",
+    "1.0",
+    "--seed",
+    "42",
+    "--reward_config",
+    "blocks_stack_v2",
+    "--convert_model_dtype",
+    "--offload_model",
+    "false",
+    "--learning_rate",
+    "1e-5",
+    "--weight_decay",
+    "0.01",
+    "--max_grad_norm",
+    "2.0",
+    "--checkpointing_steps",
+    "10",
+    "--lora_rank",
+    "64",
+    "--lora_alpha",
+    "64",
+    "--wandb_project",
+    "Corrective-Reflow",
+    "--wandb_run_name",
+    "creflow_stack_blocks_three"
+  ],
+  "program":  "/mnt/bn/themis/yijiangli/project/EmbodiedVideoRL/fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "codePath":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "git":  {
+    "remote":  "git@github.com:VincentNi0107/EmbodiedVideoRL.git",
+    "commit":  "d5e0e14ab1e94676078e078720b47c2281e8446e"
+  },
+  "email":  "1163051845@qq.com",
+  "root":  "data/outputs/creflow_stack_blocks_three",
+  "host":  "n124-106-114",
+  "username":  "tiger",
+  "executable":  "/usr/bin/python",
+  "codePathLocal":  "fastvideo/train_creflow_wan_2_2_ti2v.py",
+  "cpu_count":  104,
+  "cpu_count_logical":  208,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1055740600320",
+      "used":  "290213711872"
+    }
+  },
+  "memory":  {
+    "total":  "1977660334080"
+  },
+  "cpu":  {
+    "count":  104,
+    "countLogical":  208
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper"
+    }
+  ],
+  "cudaVersion":  "12.9"
+}