| model: |
| video_backbone: |
| encoder: |
| name: wan_vae |
| model_path: /mnt/data/wangyuran/Wan2.2-TI2V-5B |
| name: wan22_ti2v_5b |
| model_path: /mnt/data/wangyuran/Wan2.2-TI2V-5B |
| from_scratch: false |
| shift_video: 5.0 |
| components: |
| - attr: dit |
| model_class: openwam.model.video_backbone.wan.models.dit.WanModel |
| extra_kwargs: |
| has_image_input: false |
| patch_size: |
| - 1 |
| - 2 |
| - 2 |
| in_dim: 48 |
| dim: 3072 |
| ffn_dim: 14336 |
| freq_dim: 256 |
| text_dim: 4096 |
| out_dim: 48 |
| num_heads: 24 |
| num_layers: 30 |
| eps: 1.0e-06 |
| seperated_timestep: true |
| require_clip_embedding: false |
| require_vae_embedding: false |
| fuse_vae_embedding_in_latents: true |
| - attr: vae |
| model_class: openwam.model.video_backbone.wan.models.vae.WanVideoVAE38 |
| extra_kwargs: {} |
| - attr: text_encoder |
| model_class: openwam.model.video_backbone.wan.models.text_encoder.WanTextEncoder |
| extra_kwargs: {} |
| tokenizer: |
| class: openwam.model.video_backbone.wan.models.text_encoder.HuggingfaceTokenizer |
| attr: tokenizer |
| subdir: tokenizer/google/umt5-xxl |
| path_kwarg: name |
| kwargs: |
| seq_len: 512 |
| clean: whitespace |
| action_backbone: |
| dim: 1024 |
| ffn_dim: 4096 |
| shift_action: 5.0 |
| freeze: |
| - video_backbone.text_encoder |
| - video_backbone.reason1 |
| - video_backbone.vae |
| - video_backbone.image_encoder |
| - video_backbone.video_encoder |
| architecture: |
| framework: dual_system |
| variant: joint_self_attn |
| action_dim: 80 |
| use_proprioception: true |
| state_dim: 80 |
| bridge_layers: null |
| bridge_interval: 1 |
| mot_checkpoint_mixed_attn: true |
| attention_mask_mode: action_sees_video |
| video_attention_mask_mode: first_frame_causal |
| detach_bridge: false |
| idm_video_cond_noise_prob: 0.5 |
| dataloader: |
| datasets: |
| robocoin: |
| type: robocoin |
| dataset_dir: /mnt/data/wangyuran/RoboCOIN |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| multiview: true |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| normalize_mode: quantile |
| enable_action_supervision: true |
| unify_action: true |
| unify_action_map: |
| - 0-9 |
| - 34-43 |
| total_hours: 621.0 |
| enabled: true |
| weight: 1.0 |
| oxe_droid: |
| type: oxe_droid |
| dataset_dir: /mnt/data/wangyuran/OXE/DROID-Dataset |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| multiview: true |
| head_camera_choices: |
| - observation.images.exterior_1_left |
| - observation.images.exterior_2_left |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| normalize_mode: quantile |
| enable_action_supervision: true |
| unify_action: true |
| unify_action_map: |
| - 0-9 |
| - 34-43 |
| total_hours: 213.4 |
| enabled: true |
| weight: 1.0 |
| agibotworld: |
| type: agibotworld |
| dataset_dir: /mnt/data/wangyuran/AgiBotWorld-Beta |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| multiview: true |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| normalize_mode: quantile |
| enable_action_supervision: true |
| unify_action: true |
| total_hours: 1504.6 |
| enabled: true |
| weight: 1.0 |
| egodex: |
| type: egodex |
| dataset_dir: /mnt/data/wangyuran/EgoDex |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| target_camera: observation.images.ego |
| multiview: true |
| camera_layout: |
| - observation.images.ego |
| - __missing_left__ |
| - __missing_right__ |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| unify_action: true |
| total_hours: 536.3 |
| enabled: true |
| weight: 1.0 |
| ego4d: |
| type: ego4d |
| dataset_dir: /mnt/data/wangyuran/Ego4D |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| target_camera: observation.images.ego |
| multiview: true |
| camera_layout: |
| - observation.images.ego |
| - __missing_left__ |
| - __missing_right__ |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| unify_action: true |
| total_hours: 216.3 |
| enabled: true |
| weight: 1.0 |
| haiyu: |
| type: haiyu |
| dataset_dir: /mnt/data/wangyuran/Haiyu |
| manifest_path: null |
| rebuild_manifest: false |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| target_camera: observation.images.ego |
| multiview: true |
| camera_layout: |
| - observation.images.ego |
| - __missing_left__ |
| - __missing_right__ |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| unify_action: true |
| total_hours: 688.6 |
| enabled: true |
| weight: 1.0 |
| lightwheel: |
| type: lightwheel |
| dataset_dir: /mnt/data/wangyuran/lightwheel |
| manifest_path: null |
| rebuild_manifest: false |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| target_camera: observation.images.ego |
| multiview: true |
| camera_layout: |
| - observation.images.ego |
| - __missing_left__ |
| - __missing_right__ |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| unify_action: true |
| total_hours: 327.1 |
| enabled: true |
| weight: 1.0 |
| worldengine: |
| type: worldengine |
| dataset_dir: /mnt/data/wangyuran/WorldEngine |
| num_frames: 33 |
| video_stride: 4 |
| window_stride: 1 |
| height: 384 |
| width: 320 |
| target_camera: observation.images.ego |
| multiview: true |
| camera_layout: |
| - observation.images.ego |
| - __missing_left__ |
| - __missing_right__ |
| split: train |
| color_jitter: |
| brightness: 0.2 |
| contrast: 0.2 |
| saturation: 0.2 |
| hue: 0.0 |
| unify_action: true |
| total_hours: 392.7 |
| enabled: true |
| weight: 1.0 |
| type: mixture |
| weight_strategy: proportional |
| seed: 42 |
| training: |
| debug: false |
| learning_rate: 0.0001 |
| adam_betas: |
| - 0.9 |
| - 0.95 |
| weight_decay: 0.01 |
| max_grad_norm: 1.0 |
| num_epochs: 1 |
| max_steps: null |
| batch_size: 160 |
| gradient_accumulation_steps: 1 |
| lr_scheduler: cosine |
| warmup_ratio: 0.05 |
| lr_min_ratio: 0.01 |
| action_lr: null |
| video_lr: null |
| mixed_precision: bf16 |
| zero_stage: 2 |
| use_gradient_checkpointing: true |
| use_gradient_checkpointing_offload: false |
| initialize_model_on_cpu: false |
| offload_optimizer_device: none |
| lambda_video: 1.0 |
| lambda_action: 1.0 |
| max_timestep_boundary: 1.0 |
| min_timestep_boundary: 0.0 |
| timestep_sampling: default |
| timestep_sampling_lead: video |
| timestep_sampling_alpha: 1.0 |
| output_path: /mnt/data/wangyuran/openwam_checkpoints |
| save_steps: 2000 |
| keep_last_k_ckpts: 1 |
| finetune_ckpt_path: null |
| resume_ckpt_path: null |
| dataset_num_workers: 16 |
| project: |
| name: open-wam |
| output_dir: outputs/${now:%Y-%m-%d_%H-%M-%S} |
| seed: 42 |
| wandb: |
| project: open-wam |
| run_name: null |
| entity: null |
|
|
