| { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.structure:HoloBrain_Qwen3_5_VLConfig", |
| "class_type": "robo_orchard_lab.models.holobrain_0_5.structure:HoloBrain_Qwen3_5_VL", |
| "vlm_pretrain": "./ckpt/Qwen3.5-2B", |
| "decoder": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.action_decoder:HoloBrainActionDecoder", |
| "head": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:UpsampleHead", |
| "upsample_sizes": [ |
| 32, |
| 64 |
| ], |
| "input_dim": 384, |
| "dims": [ |
| 128, |
| 64 |
| ], |
| "norm": { |
| "type": "torch.nn.modules.normalization:RMSNorm", |
| "normalized_shape": 384 |
| }, |
| "act": { |
| "type": "torch.nn.modules.activation:SiLU", |
| "inplace": true |
| }, |
| "norm_act_idx": [ |
| 0, |
| 1 |
| ], |
| "num_output_layers": 2, |
| "out_dim": 8 |
| }, |
| "transformer_cfg": { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.action_decoder:HoloBrainDecoderTransformerConfig", |
| "img_cross_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "norm_layer": { |
| "type": "torch.nn.modules.normalization:RMSNorm", |
| "normalized_shape": 384 |
| }, |
| "ffn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:ActionFFN", |
| "embed_dims": 384, |
| "feedforward_channels": 3072, |
| "act_cfg": { |
| "type": "torch.nn.modules.activation:SiLU", |
| "inplace": true |
| } |
| }, |
| "operation_order": [ |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp", |
| "t_norm", |
| "multi_modal_attn", |
| "gate_msa", |
| "norm", |
| "scale_shift", |
| "ffn", |
| "gate_mlp" |
| ], |
| "joint_self_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:JointGraphAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "temp_cross_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "text_cross_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 512, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "temp_joint_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:TemporalJointGraphAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "timestep_norm_layer": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:AdaRMSNorm", |
| "normalized_shape": 384, |
| "condition_dims": 256, |
| "zero": true, |
| "ada_init": "identity", |
| "gate_bias": 1.0 |
| }, |
| "multi_modal_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:MultiModalAttention", |
| "embed_dims": 384, |
| "img_cross_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "text_cross_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 512, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "temp_joint_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:TemporalJointGraphAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| } |
| }, |
| "pre_norm": true |
| }, |
| "base_cfg": { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.action_decoder:HoloBrainDecoderBaseConfig", |
| "training_noise_scheduler": { |
| "type": "diffusers.schedulers.scheduling_ddpm:DDPMScheduler", |
| "num_train_timesteps": 1000, |
| "beta_schedule": "squaredcos_cap_v2", |
| "prediction_type": "sample", |
| "clip_sample": false |
| }, |
| "test_noise_scheduler": { |
| "type": "diffusers.schedulers.scheduling_dpmsolver_multistep:DPMSolverMultistepScheduler", |
| "num_train_timesteps": 1000, |
| "beta_schedule": "squaredcos_cap_v2", |
| "prediction_type": "sample" |
| }, |
| "num_inference_timesteps": 10, |
| "feature_level": [ |
| 0 |
| ], |
| "state_dims": 8, |
| "embed_dims": 384, |
| "pred_steps": 64, |
| "act_cfg": { |
| "type": "torch.nn.modules.activation:SiLU", |
| "inplace": true |
| }, |
| "num_test_traj": 1, |
| "chunk_size": 4, |
| "force_kinematics": false, |
| "use_joint_mask": true, |
| "noise_type": "local_joint", |
| "pred_scaled_joint": false, |
| "prediction_type": "relative_joint_relative_pose" |
| }, |
| "training_cfg": { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.action_decoder:HoloBrainTrainingConfig", |
| "loss": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.loss:HoloBrainActionLoss", |
| "timestep_loss_weight": 1000, |
| "parallel_loss_weight": 0.1, |
| "smooth_l1_beta": 0.04, |
| "loss_mode": "smooth_l1" |
| }, |
| "temporal_attn_drop": 0.05, |
| "num_parallel_training_sample": 4, |
| "teacher_forcing_rate": 0.02 |
| }, |
| "robot_encoder": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.robot_state_encoder:HoloBrainRobotStateEncoder", |
| "transformer_cfg": { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.robot_state_encoder:HoloBrainEncoderTransformerConfig", |
| "joint_self_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:JointGraphAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "norm_layer": { |
| "type": "torch.nn.modules.normalization:RMSNorm", |
| "normalized_shape": 384 |
| }, |
| "ffn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:ActionFFN", |
| "embed_dims": 384, |
| "feedforward_channels": 2048, |
| "act_cfg": { |
| "type": "torch.nn.modules.activation:SiLU", |
| "inplace": true |
| } |
| }, |
| "operation_order": [ |
| "norm", |
| "joint_self_attn", |
| null, |
| null, |
| "norm", |
| "ffn", |
| "norm", |
| "joint_self_attn", |
| null, |
| null, |
| "norm", |
| "ffn", |
| "norm", |
| "joint_self_attn", |
| null, |
| null, |
| "norm", |
| "ffn", |
| "norm", |
| "joint_self_attn", |
| null, |
| null, |
| "norm", |
| "ffn", |
| "norm" |
| ], |
| "temp_self_attn": { |
| "type": "robo_orchard_lab.models.holobrain_0_5.layers:RotaryAttention", |
| "embed_dims": 384, |
| "num_heads": 8, |
| "max_position_embeddings": 32, |
| "output_proj_init": "normal", |
| "output_proj_std": 0.004472135954999579 |
| }, |
| "pre_norm": true |
| }, |
| "base_cfg": { |
| "__config_type__": "robo_orchard_lab.models.holobrain_0_5.robot_state_encoder:HoloBrainEncoderBaseConfig", |
| "embed_dims": 384, |
| "state_dims": 8, |
| "act_cfg": { |
| "type": "torch.nn.modules.activation:SiLU", |
| "inplace": true |
| }, |
| "chunk_size": 1 |
| } |
| } |
| }, |
| "spatial_enhancer": { |
| "type": "robo_orchard_lab.models.bip3d.spatial_enhancer:DepthFusionSpatialEnhancer", |
| "embed_dims": 384, |
| "feature_3d_dim": 128, |
| "num_depth_layers": 2, |
| "min_depth": 0.01, |
| "max_depth": 1.2, |
| "num_depth": 128, |
| "with_feature_3d": true, |
| "loss_depth_weight": 1.0 |
| }, |
| "data_preprocessor": { |
| "type": "robo_orchard_lab.models.layers.data_preprocessors:BaseDataPreprocessor", |
| "channel_flip": true, |
| "unsqueeze_depth_channel": true, |
| "batch_transforms": [ |
| { |
| "type": "robo_orchard_lab.models.bip3d.spatial_enhancer:BatchDepthProbGTGenerator", |
| "min_depth": 0.01, |
| "max_depth": 1.2, |
| "num_depth": 128, |
| "origin_stride": 2, |
| "valid_threshold": 0.5, |
| "stride": [ |
| 32 |
| ] |
| }, |
| { |
| "type": "robo_orchard_lab.models.holobrain_0_5.structure:TextTemplate" |
| } |
| ] |
| }, |
| "backbone_3d": { |
| "type": "robo_orchard_lab.models.modules.swin_transformer:SwinTransformer", |
| "in_channels": 1, |
| "embed_dims": 32, |
| "depths": [ |
| 2, |
| 6, |
| 2 |
| ], |
| "num_heads": [ |
| 2, |
| 4, |
| 8 |
| ], |
| "window_size": 8, |
| "patch_size": 8, |
| "strides": [ |
| 8, |
| 2, |
| 2 |
| ], |
| "mlp_ratio": 4, |
| "qkv_bias": true, |
| "qk_scale": null, |
| "drop_rate": 0.0, |
| "attn_drop_rate": 0.0, |
| "out_indices": [ |
| 2 |
| ], |
| "with_cp": true, |
| "convert_weights": false |
| }, |
| "neck_3d": null, |
| "input_2d": "imgs", |
| "input_3d": "depths", |
| "freeze_vlm": false, |
| "freeze_vision": true, |
| "use_state_dict_with_vlm": true, |
| "load_vlm_checkpoint": true, |
| "save_model_with_vlm": false, |
| "num_vlm_layers": 4 |
| } |
