Add files using upload-large-folder tool

config.yaml

@@ -0,0 +1,165 @@
+run_id: 0707_WanMoTJoint_flexiv_plug_2views
+run_root_dir: ./results/Checkpoints/vla
+seed: 42
+trackers:
+- jsonl
+is_debug: false
+framework:
+  name: WanMoTJoint
+  torch_dtype: bfloat16
+  skip_dit_load_from_pretrain: false
+  concat_multi_camera: horizontal
+  view_image_size:
+  - 224
+  - 224
+  time_alignment:
+    enabled: false
+  video_model:
+    model_id: Wan-AI/Wan2.2-TI2V-5B
+    tokenizer_model_id: Wan-AI/Wan2.1-T2V-1.3B
+    tokenizer_max_len: 128
+    load_text_encoder: false
+    redirect_common_files: true
+    dit_path: playground/Pretrained_models/Wan2.2-TI2V-5B
+    vae_path: playground/Pretrained_models/Wan2.2-TI2V-5B/Wan2.2_VAE.pth
+    text_encoder_path: playground/Pretrained_models/Wan2.2-TI2V-5B/models_t5_umt5-xxl-enc-bf16.pth
+    tokenizer_path: playground/Pretrained_models/umt5-xxl
+    z_dim: 48
+    vae_dim: 160
+    enable_prompt_cache: true
+    prefer_cached_text_context: true
+    text_mask_padding_as_valid: true
+    prompt_template: 'A video recorded from a robot''s point of view executing the
+      following instruction: {task}'
+    config:
+      has_image_input: false
+      patch_size:
+      - 1
+      - 2
+      - 2
+      in_dim: 48
+      hidden_dim: 3072
+      ffn_dim: 14336
+      freq_dim: 256
+      text_dim: 4096
+      out_dim: 48
+      num_heads: 24
+      attn_head_dim: 128
+      num_layers: 30
+      eps: 1.0e-06
+      seperated_timestep: true
+      require_clip_embedding: false
+      require_vae_embedding: false
+      fuse_vae_embedding_in_latents: true
+      action_conditioned: false
+      action_group_causal_mask_mode: group_diagonal
+      video_attention_mask_mode: first_frame_causal
+      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
+    scheduler:
+      train_shift: 5.0
+      infer_shift: 5.0
+      num_train_timesteps: 1000
+    text_encoder_kwargs:
+      dim: 4096
+      dim_attn: 4096
+      dim_ffn: 10240
+      num_heads: 64
+      num_layers: 24
+      num_buckets: 32
+      shared_pos: false
+      dropout: 0.1
+  action_model:
+    action_horizon: 48
+    action_dim: 8
+    state_dim: 8
+    num_inference_steps: 10
+    model_path: results/Checkpoints/WanMoT/ActionExpert_linear_interp_Wan22_alphascale_1024hdim.pt
+    skip_load_from_pretrain: ${framework.skip_dit_load_from_pretrain}
+    supervision:
+      target: action_space
+      weight_mode: none
+    config:
+      action_dim: ${framework.action_model.action_dim}
+      hidden_dim: 1024
+      ffn_dim: 4096
+      num_heads: 24
+      attn_head_dim: 128
+      num_layers: 30
+      text_dim: ${framework.video_model.config.text_dim}
+      freq_dim: 256
+      eps: 1.0e-06
+      use_gradient_checkpointing: ${trainer.enable_gradient_checkpointing}
+    scheduler:
+      train_shift: 5.0
+      infer_shift: 5.0
+      num_train_timesteps: 1000
+      sample_mode: beta_tau
+      noise_beta_alpha: 1.5
+      noise_beta_beta: 1.0
+      noise_s: 0.999
+  tactile_model:
+    hidden_dim: 512
+    ffn_dim: 2048
+    num_inference_steps: ${framework.action_model.num_inference_steps}
+    split_step: 6
+    delay_stride: ${datasets.vla_data.future_frame_stride}
+  loss:
+    lambda_video: 1.0
+    lambda_action: 1.0
+datasets:
+  vla_data:
+    dataset_py: vla_datasets
+    data_mix: flexiv_plug_2views
+    action_horizon: ${framework.action_model.action_horizon}
+    action_pad_size: ${framework.action_model.action_dim}
+    state_pad_size: ${framework.action_model.state_dim}
+    disable_state: false
+    image_size: ${framework.view_image_size}
+    use_future_frames: true
+    num_future_frames: 12
+    future_frame_stride: 4
+    use_decord: true
+    vlm_obs_delay: 0
+    text_embedding_cache_dir: data/text_embeds_cache/flexiv
+    require_text_embedding_cache: true
+    text_context_len: ${framework.video_model.tokenizer_max_len}
+    text_cache_encoder_id: wan22ti2v5b
+    text_context_prompt_template: ${framework.video_model.prompt_template}
+    per_device_batch_size: 8
+    split_strategy: episode_ratio
+    eval_ratio: 0.02
+    split_seed: 42
+    eval_per_device_batch_size: 4
+    num_workers: 8
+trainer:
+  epochs: 100
+  max_train_steps: 20000
+  num_warmup_steps: 1000
+  save_interval: 5000
+  eval_interval: 200
+  logging_frequency: 10
+  learning_rate:
+    base: 0.0001
+  lr_scheduler_type: cosine_with_min_lr
+  scheduler_specific_kwargs:
+    min_lr: 1.0e-06
+  freeze_modules: null
+  max_grad_norm: 1.0
+  warmup_ratio: 0.05
+  weight_decay: 1.0e-05
+  gradient_clipping: 1.0
+  gradient_accumulation_steps: 2
+  knowledge_isolation: false
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  enable_compile: true
+  resume_from_checkpoint: latest
+  init_from_stage_checkpoint: null
+  optimizer:
+    name: AdamW
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    weight_decay: 0.01
+output_dir: ./results/Checkpoints/vla/0707_WanMoTJoint_flexiv_plug_2views

dataset_statistics.json

@@ -0,0 +1,92 @@
+{
+  "flexiv_tac": {
+    "action": {
+      "mean": [
+        -0.0012282146669037575,
+        -0.004907892788618862,
+        -0.00098577813814357,
+        -0.001942175575827419,
+        0.00029975388375520036,
+        0.0029154106008861887,
+        -0.0023952723103931642,
+        0.05475246071322809
+      ],
+      "std": [
+        0.017417425369337627,
+        0.06274184436860133,
+        0.014912954983149748,
+        0.06248854554811464,
+        0.017541449500550067,
+        0.07108631840033632,
+        0.043746502221009106,
+        0.028552116406501238
+      ],
+      "min": [
+        -0.18175049871206284,
+        -0.5141234695911407,
+        -0.15028800070285797,
+        -0.6381519436836243,
+        -0.20766699314117432,
+        -0.5574750006198883,
+        -0.368258997797966,
+        0.0
+      ],
+      "max": [
+        0.18356749415397644,
+        0.49552297592163086,
+        0.17738299816846848,
+        0.7097319960594177,
+        0.17820249870419502,
+        0.5664500296115875,
+        0.3530514985322952,
+        0.10110000148415565
+      ],
+      "q01": null,
+      "q99": null
+    },
+    "state": {
+      "mean": [
+        -0.002284890991328478,
+        -0.6879027064896839,
+        -0.0020833111627140584,
+        1.6262867223801076,
+        0.006774415318821826,
+        0.7430012638202609,
+        -0.02373740662075458,
+        0.05778775943003461
+      ],
+      "std": [
+        0.06983513765022062,
+        0.16648893697230482,
+        0.07089231409983328,
+        0.2606028781290864,
+        0.07359750676858885,
+        0.17608753827729615,
+        0.2688251571041512,
+        0.02789373622890512
+      ],
+      "min": [
+        -0.16750649362802505,
+        -1.1429139971733093,
+        -0.21077699959278107,
+        0.6568499803543091,
+        -0.31165000796318054,
+        -0.027830000966787338,
+        -1.3311590254306793,
+        0.006610000040382147
+      ],
+      "max": [
+        0.22471299767494202,
+        -0.12347950041294098,
+        0.23184100538492203,
+        2.2920989990234375,
+        0.27304550260305405,
+        1.3572785258293152,
+        1.2321564853191376,
+        0.10110000148415565
+      ],
+      "q01": null,
+      "q99": null
+    }
+  }
+}

final_model/pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:595b1610f8171a5a9e1b751db8b190e1222a41046b4a8c9f917f92e610fd523b
+size 13451578203