Add SmolVLA RoboTwin model metadata

smolvla_robotwin_all_data_full_finetune_bs512/config.json

@@ -0,0 +1,501 @@
+{
+  "_ROBOTWIN_DATA_ROOTS": [
+    "datasets/robotwin_clean_lerobotv2.1",
+    "datasets/robotwin_randomized_lerobotv2.1"
+  ],
+  "_ROBOTWIN_TASK_LIST": [
+    "adjust_bottle",
+    "beat_block_hammer",
+    "blocks_ranking_rgb",
+    "blocks_ranking_size",
+    "click_alarmclock",
+    "click_bell",
+    "dump_bin_bigbin",
+    "grab_roller",
+    "handover_block",
+    "handover_mic",
+    "hanging_mug",
+    "lift_pot",
+    "move_can_pot",
+    "move_pillbottle_pad",
+    "move_playingcard_away",
+    "move_stapler_pad",
+    "open_laptop",
+    "open_microwave",
+    "pick_diverse_bottles",
+    "pick_dual_bottles",
+    "place_a2b_left",
+    "place_a2b_right",
+    "place_bread_basket",
+    "place_bread_skillet",
+    "place_burger_fries",
+    "place_can_basket",
+    "place_cans_plasticbox",
+    "place_container_plate",
+    "place_dual_shoes",
+    "place_empty_cup",
+    "place_fan",
+    "place_mouse_pad",
+    "place_object_basket",
+    "place_object_scale",
+    "place_object_stand",
+    "place_phone_stand",
+    "place_shoe",
+    "press_stapler",
+    "put_bottles_dustbin",
+    "put_object_cabinet",
+    "rotate_qrcode",
+    "scan_object",
+    "shake_bottle",
+    "shake_bottle_horizontally",
+    "stack_blocks_three",
+    "stack_blocks_two",
+    "stack_bowls_three",
+    "stack_bowls_two",
+    "stamp_seal",
+    "turn_switch"
+  ],
+  "eval": {
+    "dataset": {
+      "img_keys": [
+        "cam_high",
+        "cam_left_wrist",
+        "cam_right_wrist"
+      ],
+      "transforms": [
+        {
+          "action_dim": 32,
+          "action_key": "action",
+          "norm_type": "mean_std",
+          "state_dim": 32,
+          "state_key": "proprio",
+          "type": "NormalizeStatesAndActions"
+        },
+        {
+          "add_new_line": true,
+          "type": "ParquetPrompter",
+          "use_conversation": false
+        },
+        {
+          "max_len": 48,
+          "tokenizer": {
+            "model_path": "./checkpoints/SmolVLM2-500M-Video-Instruct",
+            "type": "PretrainedTokenizer"
+          },
+          "type": "ProcessPrompts"
+        },
+        {
+          "height": 512,
+          "pad_direction": "top-left",
+          "type": "ResizeImagesWithPad",
+          "width": 512
+        },
+        {
+          "type": "SimpleNormalizeImages"
+        }
+      ],
+      "type": "PrivateInferenceDataset"
+    },
+    "denormalize_action": {
+      "action_dim": 14,
+      "norm_type": "mean_std",
+      "statistic_name": "robotwin_all",
+      "type": "DenormalizePrivateAction"
+    },
+    "eval_chunk_size": 50,
+    "model_family": "smolvla",
+    "num_trials_per_task": 100,
+    "seed": 0,
+    "task_list": [
+      "adjust_bottle",
+      "beat_block_hammer",
+      "blocks_ranking_rgb",
+      "blocks_ranking_size",
+      "click_alarmclock",
+      "click_bell",
+      "dump_bin_bigbin",
+      "grab_roller",
+      "handover_block",
+      "handover_mic",
+      "hanging_mug",
+      "lift_pot",
+      "move_can_pot",
+      "move_pillbottle_pad",
+      "move_playingcard_away",
+      "move_stapler_pad",
+      "open_laptop",
+      "open_microwave",
+      "pick_diverse_bottles",
+      "pick_dual_bottles",
+      "place_a2b_left",
+      "place_a2b_right",
+      "place_bread_basket",
+      "place_bread_skillet",
+      "place_burger_fries",
+      "place_can_basket",
+      "place_cans_plasticbox",
+      "place_container_plate",
+      "place_dual_shoes",
+      "place_empty_cup",
+      "place_fan",
+      "place_mouse_pad",
+      "place_object_basket",
+      "place_object_scale",
+      "place_object_stand",
+      "place_phone_stand",
+      "place_shoe",
+      "press_stapler",
+      "put_bottles_dustbin",
+      "put_object_cabinet",
+      "rotate_qrcode",
+      "scan_object",
+      "shake_bottle",
+      "shake_bottle_horizontally",
+      "stack_blocks_three",
+      "stack_blocks_two",
+      "stack_bowls_three",
+      "stack_bowls_two",
+      "stamp_seal",
+      "turn_switch"
+    ],
+    "task_suite_name": [
+      "clean",
+      "random"
+    ],
+    "type": "RobotwinEvalRunner",
+    "unnorm_key": "robotwin_all"
+  },
+  "inference_model": {
+    "action_in_proj": {
+      "in_dim": 32,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "action_out_proj": {
+      "in_dim": 720,
+      "out_dim": 32,
+      "type": "LinearProjector"
+    },
+    "action_time_mlp_in": {
+      "in_dim": 1440,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "action_time_mlp_out": {
+      "in_dim": 720,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "add_image_special_tokens": false,
+    "chunk_size": 50,
+    "freeze_vlm_backbone": false,
+    "llm_expert": {
+      "attention_bias": false,
+      "attention_mode": "cross_attn",
+      "head_dim": 64,
+      "hidden_act": "silu",
+      "hidden_size": 720,
+      "intermediate_size": -1,
+      "max_position_embeddings": 8192,
+      "num_attention_heads": 15,
+      "num_hidden_layers": 16,
+      "num_key_value_heads": 5,
+      "rms_norm_eps": 1e-05,
+      "self_attn_every_n_layers": 2,
+      "torch_dtype": "bfloat16",
+      "type": "SmolVLMExpert",
+      "vlm_kv_dim": 320,
+      "vocab_size": 49280
+    },
+    "max_action_dim": 32,
+    "name_mapping": {
+      "action_in_proj.projector": "model.action_in_proj",
+      "action_out_proj.projector": "model.action_out_proj",
+      "action_time_mlp_in.projector": "model.action_time_mlp_in",
+      "action_time_mlp_out.projector": "model.action_time_mlp_out",
+      "llm_expert.expert": "model.vlm_with_expert.lm_expert",
+      "state_proj.projector": "model.state_proj",
+      "vlm_backbone.vlm": "model.vlm_with_expert.vlm.model"
+    },
+    "num_steps": 10,
+    "ori_action_dim": 14,
+    "pretrained_name_or_path": "./checkpoints/smolvla_base/model.safetensors",
+    "state_proj": {
+      "in_dim": 32,
+      "out_dim": 960,
+      "type": "LinearProjector"
+    },
+    "type": "SmolVLAFlowMatching",
+    "vlm_backbone": {
+      "num_vlm_layers": 16,
+      "scale_factor": 4,
+      "text_config": {
+        "head_dim": 64,
+        "hidden_act": "silu",
+        "hidden_size": 960,
+        "intermediate_size": 2560,
+        "max_position_embeddings": 8192,
+        "num_attention_heads": 15,
+        "num_hidden_layers": 32,
+        "num_key_value_heads": 5,
+        "rms_norm_eps": 1e-05,
+        "vocab_size": 49280
+      },
+      "torch_dtype": "bfloat16",
+      "type": "SmolVLMBackbone",
+      "vision_config": {
+        "hidden_act": "gelu_pytorch_tanh",
+        "hidden_size": 768,
+        "image_size": 512,
+        "intermediate_size": 3072,
+        "layer_norm_eps": 1e-06,
+        "num_attention_heads": 12,
+        "num_hidden_layers": 12,
+        "patch_size": 16
+      }
+    }
+  },
+  "model": {
+    "action_in_proj": {
+      "in_dim": 32,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "action_out_proj": {
+      "in_dim": 720,
+      "out_dim": 32,
+      "type": "LinearProjector"
+    },
+    "action_time_mlp_in": {
+      "in_dim": 1440,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "action_time_mlp_out": {
+      "in_dim": 720,
+      "out_dim": 720,
+      "type": "LinearProjector"
+    },
+    "add_image_special_tokens": false,
+    "chunk_size": 50,
+    "freeze_vlm_backbone": false,
+    "llm_expert": {
+      "attention_bias": false,
+      "attention_mode": "cross_attn",
+      "head_dim": 64,
+      "hidden_act": "silu",
+      "hidden_size": 720,
+      "intermediate_size": -1,
+      "max_position_embeddings": 8192,
+      "num_attention_heads": 15,
+      "num_hidden_layers": 16,
+      "num_key_value_heads": 5,
+      "rms_norm_eps": 1e-05,
+      "self_attn_every_n_layers": 2,
+      "torch_dtype": "bfloat16",
+      "type": "SmolVLMExpert",
+      "vlm_kv_dim": 320,
+      "vocab_size": 49280
+    },
+    "max_action_dim": 32,
+    "name_mapping": {
+      "action_in_proj.projector": "model.action_in_proj",
+      "action_out_proj.projector": "model.action_out_proj",
+      "action_time_mlp_in.projector": "model.action_time_mlp_in",
+      "action_time_mlp_out.projector": "model.action_time_mlp_out",
+      "llm_expert.expert": "model.vlm_with_expert.lm_expert",
+      "state_proj.projector": "model.state_proj",
+      "vlm_backbone.vlm": "model.vlm_with_expert.vlm.model"
+    },
+    "num_steps": 10,
+    "ori_action_dim": 14,
+    "pretrained_name_or_path": "./checkpoints/smolvla_base/model.safetensors",
+    "state_proj": {
+      "in_dim": 32,
+      "out_dim": 960,
+      "type": "LinearProjector"
+    },
+    "type": "SmolVLAFlowMatching",
+    "vlm_backbone": {
+      "num_vlm_layers": 16,
+      "scale_factor": 4,
+      "text_config": {
+        "head_dim": 64,
+        "hidden_act": "silu",
+        "hidden_size": 960,
+        "intermediate_size": 2560,
+        "max_position_embeddings": 8192,
+        "num_attention_heads": 15,
+        "num_hidden_layers": 32,
+        "num_key_value_heads": 5,
+        "rms_norm_eps": 1e-05,
+        "vocab_size": 49280
+      },
+      "torch_dtype": "bfloat16",
+      "type": "SmolVLMBackbone",
+      "vision_config": {
+        "hidden_act": "gelu_pytorch_tanh",
+        "hidden_size": 768,
+        "image_size": 512,
+        "intermediate_size": 3072,
+        "layer_norm_eps": 1e-06,
+        "num_attention_heads": 12,
+        "num_hidden_layers": 12,
+        "patch_size": 16
+      }
+    }
+  },
+  "runner": {
+    "change_key_name": false,
+    "collator": {
+      "keys": [
+        "states",
+        "timestamp",
+        "images",
+        "img_masks",
+        "lang_tokens",
+        "lang_masks",
+        "actions",
+        "action_masks"
+      ],
+      "meta_keys": [
+        "task_description",
+        "prompt",
+        "info",
+        "stats"
+      ],
+      "type": "DictCollator"
+    },
+    "deterministic_algorithms": true,
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "grad_accumulation_steps": 1,
+    "lr_scheduler": {
+      "min_lr": 2.5e-06,
+      "type": "linear-warmup+cosine-decay",
+      "warmup_ratio": 0.03
+    },
+    "max_epochs": 5,
+    "max_grad_norm": 10.0,
+    "max_keep_ckpts": 3,
+    "max_steps": null,
+    "metric": {
+      "active_trackers": [
+        "jsonl",
+        "wandb"
+      ],
+      "run_dir": "work_dirs",
+      "type": "VLAMetric",
+      "window_size": 1
+    },
+    "mixed_precision_dtype": "bf16",
+    "optimizer": {
+      "lr": 0.0001,
+      "type": "AdamW",
+      "weight_decay": 0.01
+    },
+    "sampler": null,
+    "save_iter_interval": 10000,
+    "seed": 0,
+    "sharding_strategy": "full-shard",
+    "tokenizer": {
+      "model_path": "./checkpoints/SmolVLM2-500M-Video-Instruct",
+      "type": "PretrainedTokenizer"
+    },
+    "type": "FSDPTrainRunner"
+  },
+  "train_dataloader": {
+    "dataset": {
+      "datasets": [
+        {
+          "action_key": "action",
+          "action_window_size": 50,
+          "data_root_path": [
+            "datasets/robotwin_clean_lerobotv2.1",
+            "datasets/robotwin_randomized_lerobotv2.1"
+          ],
+          "statistic_name": "robotwin_all",
+          "transforms": [
+            {
+              "name_mappings": {
+                "actions": [
+                  "actions"
+                ],
+                "observation.state": [
+                  "states"
+                ]
+              },
+              "parquet_keys": [
+                "observation.state",
+                "timestamp",
+                "actions",
+                "info",
+                "stats",
+                "action_masks"
+              ],
+              "type": "ProcessParquetInputs",
+              "video_backend": "pyav",
+              "video_keys": [
+                "observation.images.cam_high",
+                "observation.images.cam_left_wrist",
+                "observation.images.cam_right_wrist"
+              ]
+            },
+            {
+              "action_dim": 32,
+              "action_key": "action",
+              "norm_type": "mean_std",
+              "state_dim": 32,
+              "state_key": "proprio",
+              "type": "NormalizeStatesAndActions"
+            },
+            {
+              "add_new_line": true,
+              "type": "ParquetPrompter",
+              "use_conversation": false
+            },
+            {
+              "max_len": 48,
+              "tokenizer": {
+                "model_path": "./checkpoints/SmolVLM2-500M-Video-Instruct",
+                "type": "PretrainedTokenizer"
+              },
+              "type": "ProcessPrompts"
+            },
+            {
+              "height": 512,
+              "pad_direction": "top-left",
+              "type": "ResizeImagesWithPad",
+              "width": 512
+            },
+            {
+              "type": "SimpleNormalizeImages"
+            }
+          ],
+          "type": "ParquetDataset",
+          "use_delta": false,
+          "window_start_idx": 0
+        }
+      ],
+      "name_mappings": {
+        "action": [
+          "action"
+        ],
+        "observation.state": [
+          "proprio"
+        ]
+      },
+      "reshuffle_each_epoch": true,
+      "seed": 0,
+      "statistic_keys": [
+        "observation.state",
+        "action",
+        "timestamp"
+      ],
+      "statistic_name": "robotwin_all",
+      "type": "DistributedRepeatingDataset"
+    },
+    "per_device_batch_size": 32,
+    "per_device_num_workers": 4
+  }
+}