Add files using upload-large-folder tool

config.yaml

@@ -0,0 +1,157 @@
+run_id: 0625_WanMoTMultiViewJoint_flexiv_3views
+run_root_dir: ./results/Checkpoints/vla
+seed: 42
+trackers:
+- jsonl
+is_debug: false
+framework:
+  name: WanMoTMultiViewJoint
+  torch_dtype: bfloat16
+  skip_dit_load_from_pretrain: false
+  concat_multi_camera: horizontal
+  view_image_size:
+  - 224
+  - 224
+  video_model:
+    model_id: Wan-AI/Wan2.2-TI2V-5B
+    tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
+    tokenizer_max_len: 128
+    load_text_encoder: false
+    redirect_common_files: true
+    dit_path: playground/Pretrained_models/Wan2.2-TI2V-5B
+    vae_path: playground/Pretrained_models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+    text_encoder_path: playground/Pretrained_models/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
+    tokenizer_path: playground/Pretrained_models/umt5-xxl
+    z_dim: 48
+    vae_dim: 160
+    enable_prompt_cache: true
+    prefer_cached_text_context: true
+    text_mask_padding_as_valid: true
+    prompt_template: 'A video recorded from a robot''s point of view executing the
+      following instruction: {task}'
+    config:
+      has_image_input: false
+      patch_size:
+      - 1
+      - 2
+      - 2
+      in_dim: 48
+      hidden_dim: 3072
+      ffn_dim: 14336
+      freq_dim: 256
+      text_dim: 4096
+      out_dim: 48
+      num_heads: 24
+      attn_head_dim: 128
+      num_layers: 30
+      eps: 1.0e-06
+      seperated_timestep: true
+      require_clip_embedding: false
+      require_vae_embedding: false
+      fuse_vae_embedding_in_latents: true
+      action_conditioned: false
+      action_group_causal_mask_mode: group_diagonal
+      video_attention_mask_mode: first_frame_causal
+      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
+    scheduler:
+      train_shift: 5.0
+      infer_shift: 5.0
+      num_train_timesteps: 1000
+    text_encoder_kwargs:
+      dim: 4096
+      dim_attn: 4096
+      dim_ffn: 10240
+      num_heads: 64
+      num_layers: 24
+      num_buckets: 32
+      shared_pos: false
+      dropout: 0.1
+  action_model:
+    action_horizon: 48
+    action_dim: 8
+    state_dim: 8
+    num_inference_steps: 10
+    model_path: results/Checkpoints/WanMoT/ActionExpert_linear_interp_Wan22_alphascale_1024hdim.pt
+    skip_load_from_pretrain: ${framework.skip_dit_load_from_pretrain}
+    supervision:
+      target: action_space
+      weight_mode: none
+    config:
+      action_dim: ${framework.action_model.action_dim}
+      hidden_dim: 1024
+      ffn_dim: 4096
+      num_heads: 24
+      attn_head_dim: 128
+      num_layers: 30
+      text_dim: ${framework.video_model.config.text_dim}
+      freq_dim: 256
+      eps: 1.0e-06
+      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
+    scheduler:
+      train_shift: 5.0
+      infer_shift: 5.0
+      num_train_timesteps: 1000
+      sample_mode: beta_tau
+      noise_beta_alpha: 1.5
+      noise_beta_beta: 1.0
+      noise_s: 0.999
+  loss:
+    lambda_video: 1.0
+    lambda_action: 1.0
+datasets:
+  vla_data:
+    dataset_py: vla_datasets
+    data_mix: flexiv_3views
+    action_horizon: ${framework.action_model.action_horizon}
+    action_pad_size: ${framework.action_model.action_dim}
+    state_pad_size: ${framework.action_model.state_dim}
+    disable_state: false
+    image_size: ${framework.view_image_size}
+    use_future_frames: true
+    num_future_frames: 12
+    future_frame_stride: 4
+    use_decord: true
+    vlm_obs_delay: 0
+    text_embedding_cache_dir: data/text_embeds_cache/flexiv
+    require_text_embedding_cache: true
+    text_context_len: ${framework.video_model.tokenizer_max_len}
+    text_cache_encoder_id: wan22ti2v5b
+    text_context_prompt_template: ${framework.video_model.prompt_template}
+    per_device_batch_size: 8
+    split_strategy: episode_ratio
+    eval_ratio: 0.02
+    split_seed: 42
+    eval_per_device_batch_size: 4
+    num_workers: 8
+trainer:
+  epochs: 100
+  max_train_steps: 20000
+  num_warmup_steps: 1000
+  save_interval: 5000
+  eval_interval: 200
+  logging_frequency: 10
+  learning_rate:
+    base: 0.0001
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: null
+  max_grad_norm: 1.0
+  warmup_ratio: 0.05
+  weight_decay: 1.0e-05
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 1
+  knowledge_isolation: false
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  enable_compile: true
+  resume_from_checkpoint: latest
+  init_from_stage_checkpoint: null
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 0.01
+output_dir: ./results/Checkpoints/vla/0625_WanMoTMultiViewJoint_flexiv_3views

dataset_statistics.json

@@ -0,0 +1,92 @@
+{
+  "flexiv_tac": {
+    "action": {
+      "mean": [
+        -0.0021471975889416116,
+        -0.006510055349735526,
+        -0.0013380912320339093,
+        -0.005949666203842312,
+        0.0009088097982147607,
+        0.00024194774618673394,
+        -0.0027595721283229175,
+        0.05042095424915953
+      ],
+      "std": [
+        0.02171544294282339,
+        0.07145401429320582,
+        0.019291937336467967,
+        0.07751492140178613,
+        0.02085302200153759,
+        0.08339417326473307,
+        0.05424014822946641,
+        0.026841927188824458
+      ],
+      "min": [
+        -0.20565299689769745,
+        -0.561244010925293,
+        -0.18788300454616547,
+        -0.6890000104904175,
+        -0.27163398265838623,
+        -0.5268030166625977,
+        -0.42483702301979065,
+        0.0
+      ],
+      "max": [
+        0.1963060051202774,
+        0.5743389129638672,
+        0.21102400124073029,
+        0.7800420522689819,
+        0.22932401299476624,
+        0.6647109985351562,
+        0.3987810015678406,
+        0.10006000101566315
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "state": {
+      "mean": [
+        0.004597892502169826,
+        -0.6628407539214973,
+        0.013475039120146451,
+        1.661976362304327,
+        -0.003530769040758804,
+        0.7484176425966574,
+        0.05153932248836374,
+        0.05381028527533211
+      ],
+      "std": [
+        0.0855862864237747,
+        0.18022102250382083,
+        0.0862308077121702,
+        0.2924328309708477,
+        0.08576178857500444,
+        0.1901975361172308,
+        0.32797629752918706,
+        0.02645007695338555
+      ],
+      "min": [
+        -0.17071199417114258,
+        -1.1214179992675781,
+        -0.22675900161266327,
+        0.6467589735984802,
+        -0.3490380048751831,
+        -0.13446000218391418,
+        -1.7821190357208252,
+        0.013220000080764294
+      ],
+      "max": [
+        0.26929599046707153,
+        -0.09738600254058838,
+        0.26130300760269165,
+        2.3931961059570312,
+        0.30517899990081787,
+        1.4650850296020508,
+        1.6099319458007812,
+        0.10006000101566315
+      ],
+      "q01": null,
+      "q99": null
+    }
+  }
+}

final_model/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f37f0b745e8f5d2638f0942efe3b86d3684c982369d36781e3a5ba22e863eafc
+size 15736637467