Add DiT4DiT RoboTwin model metadata

.gitattributes

@@ -73,3 +73,4 @@ dit4dit_robodojo_full_finetune_bs128/tokenizer/tokenizer.json filter=lfs diff=lf
 pi0_paligemma_robotwin_all_data_full_finetune_bs128/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 pi05_paligemma_robotwin_all_data_full_finetune_bs128/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 fastwam_robotwin_all_data_full_finetune_bs256/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

dit4dit_robotwin_all_data_full_finetune_bs256/config.json

@@ -0,0 +1,549 @@
+{
+  "_DATA_ROOTS": [
+    "datasets/robotwin_clean_lerobotv2.1",
+    "datasets/robotwin_randomized_lerobotv2.1"
+  ],
+  "_TASK_LIST": [
+    "adjust_bottle",
+    "beat_block_hammer",
+    "blocks_ranking_rgb",
+    "blocks_ranking_size",
+    "click_alarmclock",
+    "click_bell",
+    "dump_bin_bigbin",
+    "grab_roller",
+    "handover_block",
+    "handover_mic",
+    "hanging_mug",
+    "lift_pot",
+    "move_can_pot",
+    "move_pillbottle_pad",
+    "move_playingcard_away",
+    "move_stapler_pad",
+    "open_laptop",
+    "open_microwave",
+    "pick_diverse_bottles",
+    "pick_dual_bottles",
+    "place_a2b_left",
+    "place_a2b_right",
+    "place_bread_basket",
+    "place_bread_skillet",
+    "place_burger_fries",
+    "place_can_basket",
+    "place_cans_plasticbox",
+    "place_container_plate",
+    "place_dual_shoes",
+    "place_empty_cup",
+    "place_fan",
+    "place_mouse_pad",
+    "place_object_basket",
+    "place_object_scale",
+    "place_object_stand",
+    "place_phone_stand",
+    "place_shoe",
+    "press_stapler",
+    "put_bottles_dustbin",
+    "put_object_cabinet",
+    "rotate_qrcode",
+    "scan_object",
+    "shake_bottle",
+    "shake_bottle_horizontally",
+    "stack_blocks_three",
+    "stack_blocks_two",
+    "stack_bowls_three",
+    "stack_bowls_two",
+    "stamp_seal",
+    "turn_switch"
+  ],
+  "_TOKENIZER_PATH": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+  "eval": {
+    "dataset": {
+      "img_keys": [
+        "cam_high",
+        "cam_left_wrist",
+        "cam_right_wrist"
+      ],
+      "inject_model_path": false,
+      "transforms": [
+        {
+          "input_key": "task_description",
+          "remove_input_key": true,
+          "tokenizer": {
+            "model_max_length": 512,
+            "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+            "type": "PretrainedTokenizer"
+          },
+          "type": "ProcessCosmos25Prompt"
+        },
+        {
+          "backend": "torch",
+          "height": 224,
+          "output_layout": "flattened_chw",
+          "scale_divisor": 255.0,
+          "type": "ResizeImages",
+          "width": 224
+        },
+        {
+          "means": [
+            0.5,
+            0.5,
+            0.5
+          ],
+          "stds": [
+            0.5,
+            0.5,
+            0.5
+          ],
+          "type": "NormalizeImages"
+        },
+        {
+          "bottom_height_ratio": 0.5,
+          "bottom_views": [
+            1,
+            2
+          ],
+          "combine_view_masks": true,
+          "frame_window_size": 1,
+          "num_views": 3,
+          "tile_direction": "top_bottom_pair",
+          "top_view": 0,
+          "type": "PrepareVideo"
+        },
+        {
+          "backend": "torch",
+          "expand_axis": 0,
+          "interleave": true,
+          "keys": [
+            "states"
+          ],
+          "target_dims": {
+            "states": 32
+          },
+          "type": "SinCosKeys"
+        },
+        {
+          "action_dim": 16,
+          "action_key": "action",
+          "norm_type": "min_max",
+          "normalization_epsilon": 1e-06,
+          "output_dtype": "float32",
+          "state_dim": null,
+          "state_key": "proprio",
+          "state_norm_type": "none",
+          "type": "NormalizeStatesAndActions"
+        },
+        {
+          "action_horizon": 32,
+          "state_history_length": 1,
+          "type": "PrepareStateActionTargets",
+          "valid_action_dim": 14
+        }
+      ],
+      "type": "PrivateInferenceDataset"
+    },
+    "denormalize_action": {
+      "action_dim": 14,
+      "norm_type": "min_max",
+      "statistic_name": "robotwin_all",
+      "type": "DenormalizePrivateAction"
+    },
+    "eval_chunk_size": 24,
+    "instruction_type": "unseen",
+    "mixed_precision_dtype": "bf16",
+    "model_family": "dit4dit",
+    "num_trials_per_task": 100,
+    "seed": 7,
+    "task_list": [
+      "adjust_bottle",
+      "beat_block_hammer",
+      "blocks_ranking_rgb",
+      "blocks_ranking_size",
+      "click_alarmclock",
+      "click_bell",
+      "dump_bin_bigbin",
+      "grab_roller",
+      "handover_block",
+      "handover_mic",
+      "hanging_mug",
+      "lift_pot",
+      "move_can_pot",
+      "move_pillbottle_pad",
+      "move_playingcard_away",
+      "move_stapler_pad",
+      "open_laptop",
+      "open_microwave",
+      "pick_diverse_bottles",
+      "pick_dual_bottles",
+      "place_a2b_left",
+      "place_a2b_right",
+      "place_bread_basket",
+      "place_bread_skillet",
+      "place_burger_fries",
+      "place_can_basket",
+      "place_cans_plasticbox",
+      "place_container_plate",
+      "place_dual_shoes",
+      "place_empty_cup",
+      "place_fan",
+      "place_mouse_pad",
+      "place_object_basket",
+      "place_object_scale",
+      "place_object_stand",
+      "place_phone_stand",
+      "place_shoe",
+      "press_stapler",
+      "put_bottles_dustbin",
+      "put_object_cabinet",
+      "rotate_qrcode",
+      "scan_object",
+      "shake_bottle",
+      "shake_bottle_horizontally",
+      "stack_blocks_three",
+      "stack_blocks_two",
+      "stack_bowls_three",
+      "stack_bowls_two",
+      "stamp_seal",
+      "turn_switch"
+    ],
+    "task_suite_name": [
+      "clean",
+      "random"
+    ],
+    "type": "RobotwinEvalRunner",
+    "unnorm_key": "robotwin_all"
+  },
+  "inference_model": {
+    "freeze_vlm_backbone": false,
+    "init_empty_weights": true,
+    "repeated_diffusion_steps": 4,
+    "type": "DiT4DiTVLA",
+    "vla_head": {
+      "action_dim": 16,
+      "action_horizon": 32,
+      "action_model_type": "DiT-B",
+      "add_pos_embed": true,
+      "diffusion_model_cfg": {
+        "cross_attention_dim": 2048,
+        "dropout": 0.2,
+        "final_dropout": true,
+        "interleave_self_attention": true,
+        "norm_type": "ada_norm",
+        "num_layers": 16,
+        "output_dim": 2560,
+        "positional_embeddings": null
+      },
+      "future_action_window_size": 31,
+      "hidden_size": 2560,
+      "max_seq_len": 1024,
+      "noise_beta_alpha": 1.5,
+      "noise_beta_beta": 1.0,
+      "noise_s": 0.999,
+      "num_inference_timesteps": 4,
+      "num_timestep_buckets": 1000,
+      "ori_action_dim": 14,
+      "state_dim": 32,
+      "type": "DiT4DiTActionHead"
+    },
+    "vlm_backbone": {
+      "base_model": "./checkpoints/Cosmos-Predict2.5-2B",
+      "conditional_frame_timestep": 0.0001,
+      "detach_hidden_states": true,
+      "extract_layer": 17,
+      "fixed_seed": null,
+      "flow_matching_high_sigma_min": null,
+      "flow_matching_high_sigma_ratio": null,
+      "flow_matching_time_distribution": "uniform",
+      "frozen_submodules": [
+        "text_encoder",
+        "vae"
+      ],
+      "fsdp_min_num_params": 0,
+      "future_loss_type": "flow_matching",
+      "load_pretrained_weights": false,
+      "local_files_only": true,
+      "num_frames_out": 9,
+      "num_inference_steps": 1,
+      "revision": "diffusers/base/post-trained",
+      "split_future_frames": true,
+      "torch_dtype": "bf16",
+      "trainable": true,
+      "type": "Cosmos25Backbone"
+    }
+  },
+  "model": {
+    "freeze_vlm_backbone": false,
+    "repeated_diffusion_steps": 4,
+    "type": "DiT4DiTVLA",
+    "vla_head": {
+      "action_dim": 16,
+      "action_horizon": 32,
+      "action_model_type": "DiT-B",
+      "add_pos_embed": true,
+      "diffusion_model_cfg": {
+        "cross_attention_dim": 2048,
+        "dropout": 0.2,
+        "final_dropout": true,
+        "interleave_self_attention": true,
+        "norm_type": "ada_norm",
+        "num_layers": 16,
+        "output_dim": 2560,
+        "positional_embeddings": null
+      },
+      "future_action_window_size": 31,
+      "hidden_size": 2560,
+      "max_seq_len": 1024,
+      "noise_beta_alpha": 1.5,
+      "noise_beta_beta": 1.0,
+      "noise_s": 0.999,
+      "num_inference_timesteps": 4,
+      "num_timestep_buckets": 1000,
+      "ori_action_dim": 14,
+      "state_dim": 32,
+      "type": "DiT4DiTActionHead"
+    },
+    "vlm_backbone": {
+      "base_model": "./checkpoints/Cosmos-Predict2.5-2B",
+      "conditional_frame_timestep": 0.0001,
+      "detach_hidden_states": true,
+      "extract_layer": 17,
+      "fixed_seed": null,
+      "flow_matching_high_sigma_min": null,
+      "flow_matching_high_sigma_ratio": null,
+      "flow_matching_time_distribution": "uniform",
+      "frozen_submodules": [
+        "text_encoder",
+        "vae"
+      ],
+      "fsdp_min_num_params": 0,
+      "future_loss_type": "flow_matching",
+      "local_files_only": true,
+      "num_frames_out": 9,
+      "num_inference_steps": 1,
+      "revision": "diffusers/base/post-trained",
+      "split_future_frames": true,
+      "torch_dtype": "bf16",
+      "trainable": true,
+      "type": "Cosmos25Backbone"
+    }
+  },
+  "runner": {
+    "change_key_name": false,
+    "collator": {
+      "keys": [
+        "states",
+        "timestamp",
+        "images",
+        "img_masks",
+        "actions",
+        "action_masks",
+        "frame_masks",
+        "lang_tokens",
+        "lang_masks"
+      ],
+      "meta_keys": [
+        "info",
+        "stats"
+      ],
+      "type": "DictCollator"
+    },
+    "enable_gradient_checkpointing": false,
+    "enable_mixed_precision_training": true,
+    "grad_accumulation_steps": 1,
+    "lr_scheduler": {
+      "min_lr": 5e-07,
+      "type": "linear-warmup+cosine-decay",
+      "warmup_ratio": 0.05
+    },
+    "max_epochs": 3,
+    "max_grad_norm": 1.0,
+    "max_keep_ckpts": 3,
+    "max_steps": null,
+    "metric": {
+      "active_trackers": [
+        "jsonl",
+        "wandb"
+      ],
+      "grad_accumulation_steps": 1,
+      "run_dir": "work_dirs",
+      "type": "VLAMetric",
+      "window_size": 100
+    },
+    "mixed_precision_dtype": "bf16",
+    "optimizer": {
+      "betas": [
+        0.9,
+        0.95
+      ],
+      "eps": 1e-08,
+      "lr": 3e-05,
+      "paramwise_learning_rate": {
+        "vla_head": 0.0001,
+        "vlm_backbone.transformer": 1e-05
+      },
+      "type": "AdamW",
+      "weight_decay": 1e-08,
+      "weight_decay_all_params": true
+    },
+    "pre_fsdp_param_dtype": "fp32",
+    "reduce_in_full_precision": false,
+    "sampler": null,
+    "save_epoch_interval": 1,
+    "save_iter_interval": 10000000,
+    "seed": 42,
+    "sharding_strategy": "global-shard-grad-op",
+    "tokenizer": {
+      "model_max_length": 512,
+      "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+      "type": "PretrainedTokenizer"
+    },
+    "type": "FSDPTrainRunner"
+  },
+  "train_dataloader": {
+    "dataset": {
+      "auto_compute_statistics": {
+        "profile": "absolute"
+      },
+      "dataset_statistics_path": "/mnt/data/cpfs/mnt/data/william/robotwin/work_dirs/dit4dit_robotwin_horizion32_exec24/dataset_statistics.json",
+      "datasets": {
+        "action_key": "action",
+        "action_window_size": 32,
+        "data_root_path": [
+          "datasets/robotwin_clean_lerobotv2.1",
+          "datasets/robotwin_randomized_lerobotv2.1"
+        ],
+        "frame_sample_stride": 4,
+        "frame_window_size": 9,
+        "statistic_name": "robotwin_all",
+        "supervise_terminal_padding": true,
+        "transforms": [
+          {
+            "name_mappings": {
+              "actions": [
+                "actions"
+              ],
+              "observation.state": [
+                "states"
+              ]
+            },
+            "parquet_keys": [
+              "observation.state",
+              "timestamp",
+              "actions",
+              "info",
+              "stats",
+              "action_masks"
+            ],
+            "type": "ProcessParquetInputs",
+            "video_backend": "pyav",
+            "video_keys": [
+              "observation.images.cam_high",
+              "observation.images.cam_left_wrist",
+              "observation.images.cam_right_wrist"
+            ]
+          },
+          {
+            "input_key": "task_description",
+            "remove_input_key": true,
+            "tokenizer": {
+              "model_max_length": 512,
+              "model_path": "./checkpoints/Cosmos-Predict2.5-2B/tokenizer",
+              "type": "PretrainedTokenizer"
+            },
+            "type": "ProcessCosmos25Prompt"
+          },
+          {
+            "backend": "torch",
+            "height": 224,
+            "output_layout": "flattened_chw",
+            "scale_divisor": 255.0,
+            "type": "ResizeImages",
+            "width": 224
+          },
+          {
+            "means": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "stds": [
+              0.5,
+              0.5,
+              0.5
+            ],
+            "type": "NormalizeImages"
+          },
+          {
+            "bottom_height_ratio": 0.5,
+            "bottom_views": [
+              1,
+              2
+            ],
+            "combine_view_masks": true,
+            "frame_window_size": 9,
+            "num_views": 3,
+            "tile_direction": "top_bottom_pair",
+            "top_view": 0,
+            "type": "PrepareVideo"
+          },
+          {
+            "backend": "torch",
+            "expand_axis": 0,
+            "interleave": true,
+            "keys": [
+              "states"
+            ],
+            "target_dims": {
+              "states": 32
+            },
+            "type": "SinCosKeys"
+          },
+          {
+            "action_dim": 16,
+            "action_key": "action",
+            "norm_type": "min_max",
+            "normalization_epsilon": 1e-06,
+            "output_dtype": "float32",
+            "state_dim": null,
+            "state_key": "proprio",
+            "state_norm_type": "none",
+            "type": "NormalizeStatesAndActions"
+          },
+          {
+            "action_horizon": 32,
+            "state_history_length": 1,
+            "type": "PrepareStateActionTargets",
+            "valid_action_dim": 14
+          }
+        ],
+        "type": "ParquetDataset",
+        "use_delta": false,
+        "window_start_idx": 0
+      },
+      "name_mappings": {
+        "action": [
+          "action"
+        ],
+        "observation.state": [
+          "proprio"
+        ]
+      },
+      "reshuffle_each_epoch": true,
+      "sampling_unit": "episode",
+      "sampling_weights": [
+        1.0,
+        1.0
+      ],
+      "seed": 42,
+      "shuffle": true,
+      "statistic_keys": [
+        "observation.state",
+        "timestamp",
+        "action"
+      ],
+      "statistic_name": "robotwin_all",
+      "type": "DistributedBalancedRepeatingDataset"
+    },
+    "per_device_batch_size": 8,
+    "per_device_num_workers": 4
+  }
+}

dit4dit_robotwin_all_data_full_finetune_bs256/config.yaml

@@ -0,0 +1,459 @@
+_DATA_ROOTS:
+- datasets/robotwin_clean_lerobotv2.1
+- datasets/robotwin_randomized_lerobotv2.1
+_TASK_LIST:
+- adjust_bottle
+- beat_block_hammer
+- blocks_ranking_rgb
+- blocks_ranking_size
+- click_alarmclock
+- click_bell
+- dump_bin_bigbin
+- grab_roller
+- handover_block
+- handover_mic
+- hanging_mug
+- lift_pot
+- move_can_pot
+- move_pillbottle_pad
+- move_playingcard_away
+- move_stapler_pad
+- open_laptop
+- open_microwave
+- pick_diverse_bottles
+- pick_dual_bottles
+- place_a2b_left
+- place_a2b_right
+- place_bread_basket
+- place_bread_skillet
+- place_burger_fries
+- place_can_basket
+- place_cans_plasticbox
+- place_container_plate
+- place_dual_shoes
+- place_empty_cup
+- place_fan
+- place_mouse_pad
+- place_object_basket
+- place_object_scale
+- place_object_stand
+- place_phone_stand
+- place_shoe
+- press_stapler
+- put_bottles_dustbin
+- put_object_cabinet
+- rotate_qrcode
+- scan_object
+- shake_bottle
+- shake_bottle_horizontally
+- stack_blocks_three
+- stack_blocks_two
+- stack_bowls_three
+- stack_bowls_two
+- stamp_seal
+- turn_switch
+_TOKENIZER_PATH: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+eval:
+  dataset:
+    img_keys:
+    - cam_high
+    - cam_left_wrist
+    - cam_right_wrist
+    inject_model_path: false
+    transforms:
+    - input_key: task_description
+      remove_input_key: true
+      tokenizer:
+        model_max_length: 512
+        model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+        type: PretrainedTokenizer
+      type: ProcessCosmos25Prompt
+    - backend: torch
+      height: 224
+      output_layout: flattened_chw
+      scale_divisor: 255.0
+      type: ResizeImages
+      width: 224
+    - means:
+      - 0.5
+      - 0.5
+      - 0.5
+      stds:
+      - 0.5
+      - 0.5
+      - 0.5
+      type: NormalizeImages
+    - bottom_height_ratio: 0.5
+      bottom_views:
+      - 1
+      - 2
+      combine_view_masks: true
+      frame_window_size: 1
+      num_views: 3
+      tile_direction: top_bottom_pair
+      top_view: 0
+      type: PrepareVideo
+    - backend: torch
+      expand_axis: 0
+      interleave: true
+      keys:
+      - states
+      target_dims:
+        states: 32
+      type: SinCosKeys
+    - action_dim: 16
+      action_key: action
+      norm_type: min_max
+      normalization_epsilon: 1.0e-06
+      output_dtype: float32
+      state_dim: null
+      state_key: proprio
+      state_norm_type: none
+      type: NormalizeStatesAndActions
+    - action_horizon: 32
+      state_history_length: 1
+      type: PrepareStateActionTargets
+      valid_action_dim: 14
+    type: PrivateInferenceDataset
+  denormalize_action:
+    action_dim: 14
+    norm_type: min_max
+    statistic_name: robotwin_all
+    type: DenormalizePrivateAction
+  eval_chunk_size: 24
+  instruction_type: unseen
+  mixed_precision_dtype: bf16
+  model_family: dit4dit
+  num_trials_per_task: 100
+  seed: 7
+  task_list:
+  - adjust_bottle
+  - beat_block_hammer
+  - blocks_ranking_rgb
+  - blocks_ranking_size
+  - click_alarmclock
+  - click_bell
+  - dump_bin_bigbin
+  - grab_roller
+  - handover_block
+  - handover_mic
+  - hanging_mug
+  - lift_pot
+  - move_can_pot
+  - move_pillbottle_pad
+  - move_playingcard_away
+  - move_stapler_pad
+  - open_laptop
+  - open_microwave
+  - pick_diverse_bottles
+  - pick_dual_bottles
+  - place_a2b_left
+  - place_a2b_right
+  - place_bread_basket
+  - place_bread_skillet
+  - place_burger_fries
+  - place_can_basket
+  - place_cans_plasticbox
+  - place_container_plate
+  - place_dual_shoes
+  - place_empty_cup
+  - place_fan
+  - place_mouse_pad
+  - place_object_basket
+  - place_object_scale
+  - place_object_stand
+  - place_phone_stand
+  - place_shoe
+  - press_stapler
+  - put_bottles_dustbin
+  - put_object_cabinet
+  - rotate_qrcode
+  - scan_object
+  - shake_bottle
+  - shake_bottle_horizontally
+  - stack_blocks_three
+  - stack_blocks_two
+  - stack_bowls_three
+  - stack_bowls_two
+  - stamp_seal
+  - turn_switch
+  task_suite_name:
+  - clean
+  - random
+  type: RobotwinEvalRunner
+  unnorm_key: robotwin_all
+inference_model:
+  freeze_vlm_backbone: false
+  init_empty_weights: true
+  repeated_diffusion_steps: 4
+  type: DiT4DiTVLA
+  vla_head:
+    action_dim: 16
+    action_horizon: 32
+    action_model_type: DiT-B
+    add_pos_embed: true
+    diffusion_model_cfg:
+      cross_attention_dim: 2048
+      dropout: 0.2
+      final_dropout: true
+      interleave_self_attention: true
+      norm_type: ada_norm
+      num_layers: 16
+      output_dim: 2560
+      positional_embeddings: null
+    future_action_window_size: 31
+    hidden_size: 2560
+    max_seq_len: 1024
+    noise_beta_alpha: 1.5
+    noise_beta_beta: 1.0
+    noise_s: 0.999
+    num_inference_timesteps: 4
+    num_timestep_buckets: 1000
+    ori_action_dim: 14
+    state_dim: 32
+    type: DiT4DiTActionHead
+  vlm_backbone:
+    base_model: ./checkpoints/Cosmos-Predict2.5-2B
+    conditional_frame_timestep: 0.0001
+    detach_hidden_states: true
+    extract_layer: 17
+    fixed_seed: null
+    flow_matching_high_sigma_min: null
+    flow_matching_high_sigma_ratio: null
+    flow_matching_time_distribution: uniform
+    frozen_submodules:
+    - text_encoder
+    - vae
+    fsdp_min_num_params: 0
+    future_loss_type: flow_matching
+    load_pretrained_weights: false
+    local_files_only: true
+    num_frames_out: 9
+    num_inference_steps: 1
+    revision: diffusers/base/post-trained
+    split_future_frames: true
+    torch_dtype: bf16
+    trainable: true
+    type: Cosmos25Backbone
+model:
+  freeze_vlm_backbone: false
+  repeated_diffusion_steps: 4
+  type: DiT4DiTVLA
+  vla_head:
+    action_dim: 16
+    action_horizon: 32
+    action_model_type: DiT-B
+    add_pos_embed: true
+    diffusion_model_cfg:
+      cross_attention_dim: 2048
+      dropout: 0.2
+      final_dropout: true
+      interleave_self_attention: true
+      norm_type: ada_norm
+      num_layers: 16
+      output_dim: 2560
+      positional_embeddings: null
+    future_action_window_size: 31
+    hidden_size: 2560
+    max_seq_len: 1024
+    noise_beta_alpha: 1.5
+    noise_beta_beta: 1.0
+    noise_s: 0.999
+    num_inference_timesteps: 4
+    num_timestep_buckets: 1000
+    ori_action_dim: 14
+    state_dim: 32
+    type: DiT4DiTActionHead
+  vlm_backbone:
+    base_model: ./checkpoints/Cosmos-Predict2.5-2B
+    conditional_frame_timestep: 0.0001
+    detach_hidden_states: true
+    extract_layer: 17
+    fixed_seed: null
+    flow_matching_high_sigma_min: null
+    flow_matching_high_sigma_ratio: null
+    flow_matching_time_distribution: uniform
+    frozen_submodules:
+    - text_encoder
+    - vae
+    fsdp_min_num_params: 0
+    future_loss_type: flow_matching
+    local_files_only: true
+    num_frames_out: 9
+    num_inference_steps: 1
+    revision: diffusers/base/post-trained
+    split_future_frames: true
+    torch_dtype: bf16
+    trainable: true
+    type: Cosmos25Backbone
+runner:
+  change_key_name: false
+  collator:
+    keys:
+    - states
+    - timestamp
+    - images
+    - img_masks
+    - actions
+    - action_masks
+    - frame_masks
+    - lang_tokens
+    - lang_masks
+    meta_keys:
+    - info
+    - stats
+    type: DictCollator
+  enable_gradient_checkpointing: false
+  enable_mixed_precision_training: true
+  grad_accumulation_steps: 1
+  lr_scheduler:
+    min_lr: 5.0e-07
+    type: linear-warmup+cosine-decay
+    warmup_ratio: 0.05
+  max_epochs: 3
+  max_grad_norm: 1.0
+  max_keep_ckpts: 3
+  max_steps: null
+  metric:
+    active_trackers:
+    - jsonl
+    - wandb
+    grad_accumulation_steps: 1
+    run_dir: work_dirs
+    type: VLAMetric
+    window_size: 100
+  mixed_precision_dtype: bf16
+  optimizer:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    lr: 3.0e-05
+    paramwise_learning_rate:
+      vla_head: 0.0001
+      vlm_backbone.transformer: 1.0e-05
+    type: AdamW
+    weight_decay: 1.0e-08
+    weight_decay_all_params: true
+  pre_fsdp_param_dtype: fp32
+  reduce_in_full_precision: false
+  sampler: null
+  save_epoch_interval: 1
+  save_iter_interval: 10000000
+  seed: 42
+  sharding_strategy: global-shard-grad-op
+  tokenizer:
+    model_max_length: 512
+    model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+    type: PretrainedTokenizer
+  type: FSDPTrainRunner
+train_dataloader:
+  dataset:
+    auto_compute_statistics:
+      profile: absolute
+    dataset_statistics_path: /mnt/data/cpfs/mnt/data/william/robotwin/work_dirs/dit4dit_robotwin_horizion32_exec24/dataset_statistics.json
+    datasets:
+      action_key: action
+      action_window_size: 32
+      data_root_path:
+      - datasets/robotwin_clean_lerobotv2.1
+      - datasets/robotwin_randomized_lerobotv2.1
+      frame_sample_stride: 4
+      frame_window_size: 9
+      statistic_name: robotwin_all
+      supervise_terminal_padding: true
+      transforms:
+      - name_mappings:
+          actions:
+          - actions
+          observation.state:
+          - states
+        parquet_keys:
+        - observation.state
+        - timestamp
+        - actions
+        - info
+        - stats
+        - action_masks
+        type: ProcessParquetInputs
+        video_backend: pyav
+        video_keys:
+        - observation.images.cam_high
+        - observation.images.cam_left_wrist
+        - observation.images.cam_right_wrist
+      - input_key: task_description
+        remove_input_key: true
+        tokenizer:
+          model_max_length: 512
+          model_path: ./checkpoints/Cosmos-Predict2.5-2B/tokenizer
+          type: PretrainedTokenizer
+        type: ProcessCosmos25Prompt
+      - backend: torch
+        height: 224
+        output_layout: flattened_chw
+        scale_divisor: 255.0
+        type: ResizeImages
+        width: 224
+      - means:
+        - 0.5
+        - 0.5
+        - 0.5
+        stds:
+        - 0.5
+        - 0.5
+        - 0.5
+        type: NormalizeImages
+      - bottom_height_ratio: 0.5
+        bottom_views:
+        - 1
+        - 2
+        combine_view_masks: true
+        frame_window_size: 9
+        num_views: 3
+        tile_direction: top_bottom_pair
+        top_view: 0
+        type: PrepareVideo
+      - backend: torch
+        expand_axis: 0
+        interleave: true
+        keys:
+        - states
+        target_dims:
+          states: 32
+        type: SinCosKeys
+      - action_dim: 16
+        action_key: action
+        norm_type: min_max
+        normalization_epsilon: 1.0e-06
+        output_dtype: float32
+        state_dim: null
+        state_key: proprio
+        state_norm_type: none
+        type: NormalizeStatesAndActions
+      - action_horizon: 32
+        state_history_length: 1
+        type: PrepareStateActionTargets
+        valid_action_dim: 14
+      type: ParquetDataset
+      use_delta: false
+      window_start_idx: 0
+    name_mappings:
+      action:
+      - action
+      observation.state:
+      - proprio
+    reshuffle_each_epoch: true
+    sampling_unit: episode
+    sampling_weights:
+    - 1.0
+    - 1.0
+    seed: 42
+    shuffle: true
+    statistic_keys:
+    - observation.state
+    - timestamp
+    - action
+    statistic_name: robotwin_all
+    type: DistributedBalancedRepeatingDataset
+  per_device_batch_size: 8
+  per_device_num_workers: 4

dit4dit_robotwin_all_data_full_finetune_bs256/dataset_statistics.json

@@ -0,0 +1,202 @@
+{
+  "robotwin_all": {
+    "proprio": {
+      "mean": [
+        -0.23836578982314696,
+        1.1298174457722125,
+        0.8068365622397999,
+        -0.31024205976923314,
+        0.05489046602657658,
+        -0.03797952133058626,
+        0.6655638063536832,
+        0.21664745657075993,
+        1.0790455135726966,
+        0.7875586714060262,
+        -0.3211318581147203,
+        -0.01750195667000284,
+        0.03539713811747996,
+        0.6794509068373863
+      ],
+      "std": [
+        0.4151526770142024,
+        1.004581002114008,
+        0.7768177975025599,
+        0.6732816994178228,
+        0.27081867657629666,
+        0.6197771066833538,
+        0.454651637795559,
+        0.34920797694011796,
+        1.0249604576643263,
+        0.7943722761565686,
+        0.6847884512724388,
+        0.24437404726809553,
+        0.6142938396132972,
+        0.44978408948808124
+      ],
+      "min": [
+        -7.552278995513916,
+        -0.36354875564575195,
+        -0.18577136099338531,
+        -1.956291913986206,
+        -1.6801013946533203,
+        -7.678869724273682,
+        0.0,
+        -8.539926528930664,
+        -0.6294453740119934,
+        -0.07775841653347015,
+        -2.1328067779541016,
+        -2.1285502910614014,
+        -8.228925704956055,
+        0.0
+      ],
+      "max": [
+        6.45659065246582,
+        4.179152488708496,
+        5.346591472625732,
+        1.7942548990249634,
+        1.8604620695114136,
+        5.43813943862915,
+        1.0,
+        7.142920017242432,
+        4.157068729400635,
+        5.672450065612793,
+        1.95806884765625,
+        1.5663840770721436,
+        5.278968811035156,
+        1.0
+      ],
+      "q01": [
+        -1.0126575303077698,
+        -5.257390398583084e-07,
+        -2.8215323254698887e-05,
+        -1.5635129809379578,
+        -0.6461568164825439,
+        -2.2244577407836914,
+        0.0,
+        -0.1690114441514015,
+        0.0,
+        -2.81171942333458e-05,
+        -1.6413878035545348,
+        -1.0220032143592834,
+        -1.6422083234786986,
+        0.0
+      ],
+      "q99": [
+        0.17374197930097574,
+        2.602391481399536,
+        2.4499925279617276,
+        1.351163625717163,
+        1.2348779439926147,
+        1.6033015251159668,
+        1.0,
+        0.9967232346534729,
+        2.6186706876754755,
+        2.4520904397964403,
+        1.291067123413086,
+        0.8747973608970588,
+        2.2670195341110118,
+        1.0
+      ],
+      "count": 6075103
+    },
+    "action": {
+      "mean": [
+        -0.2520778376296883,
+        1.1850720454152766,
+        0.8463635513872634,
+        -0.32170768938995004,
+        0.05836593476707814,
+        -0.047009579259007395,
+        0.6459757061337159,
+        0.23613831894636023,
+        1.1499042732131481,
+        0.8381827384224907,
+        -0.33540205675934687,
+        -0.01963055875970896,
+        0.04568921604517659,
+        0.6622663938976167
+      ],
+      "std": [
+        0.4248377447940459,
+        1.0055959718792558,
+        0.7843761483314764,
+        0.695804624051617,
+        0.2812322261529959,
+        0.6501131906725712,
+        0.46179677530217805,
+        0.3599664397795572,
+        1.0307301365174235,
+        0.8047042310821478,
+        0.7142964613182307,
+        0.25882870106807826,
+        0.6546476377063773,
+        0.45675020849497266
+      ],
+      "min": [
+        -7.552278995513916,
+        -0.36354875564575195,
+        -0.18577136099338531,
+        -1.956291913986206,
+        -1.6801013946533203,
+        -7.678869724273682,
+        0.0,
+        -8.539926528930664,
+        -0.6294453740119934,
+        -0.07775841653347015,
+        -2.1328067779541016,
+        -2.1285502910614014,
+        -8.228925704956055,
+        0.0
+      ],
+      "max": [
+        6.45659065246582,
+        4.179152488708496,
+        5.346591472625732,
+        1.7942548990249634,
+        1.8604620695114136,
+        5.43813943862915,
+        1.0,
+        7.142920017242432,
+        4.157068729400635,
+        5.672450065612793,
+        1.95806884765625,
+        1.5663840770721436,
+        5.278968811035156,
+        1.0
+      ],
+      "q01": [
+        -1.026803731918335,
+        -5.257390398583084e-07,
+        -2.8215323254698887e-05,
+        -1.5867787599563599,
+        -0.6817835569381714,
+        -2.2976155281066895,
+        0.0,
+        -0.1809745579957962,
+        0.0,
+        -2.81171942333458e-05,
+        -1.6533960103988647,
+        -1.0314260721206665,
+        -1.7116937637329102,
+        0.0
+      ],
+      "q99": [
+        0.20549810975792404,
+        2.607667922973633,
+        2.469137668609619,
+        1.3703088760375977,
+        1.2604563236236572,
+        1.6216325759887695,
+        1.0,
+        1.0141181945800781,
+        2.646111249923706,
+        2.48582124710083,
+        1.3271214962005615,
+        0.9085489511489868,
+        2.3036928176879883,
+        1.0
+      ],
+      "count": 194403296
+    }
+  }
+}

dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/chat_template.jinja

@@ -0,0 +1,7 @@
+{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system
+You are a helpful assistant.<|im_end|>
+{% endif %}<|im_start|>{{ message['role'] }}
+{% if message['content'] is string %}{{ message['content'] }}<|im_end|>
+{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>
+{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant
+{% endif %}

dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
+size 11421896

dit4dit_robotwin_all_data_full_finetune_bs256/tokenizer/tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}