Upload BOAT/BOAT_VLA

BOAT/BOAT_VLA/README.md

@@ -0,0 +1,10 @@
+---
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

BOAT/BOAT_VLA/config.json

@@ -0,0 +1,49 @@
+{
+  "action_dim": 14,
+  "action_embed_init": "normal",
+  "action_horizon": 32,
+  "action_mask_token_template": "<|action_mask|>",
+  "action_token_template": "<|action_{i}|>",
+  "action_tokenizer": {
+    "kind": "tokens",
+    "name": "boat",
+    "pretrained_path": "/scratch/xuanweiliu/bimanual_tokenizer/outputs/hopper_boat_robotwin_new/616167.hopper-m-02/train/checkpoints/best/step_300000/pretrained_model"
+  },
+  "block_ce_weighting": "block_mean",
+  "ce_vocab_scope": "full",
+  "decode_keep_k": null,
+  "decode_stop_mode": "fixed_budget",
+  "decode_temperature": null,
+  "decode_top_k": null,
+  "default_instruction": "Analyze the input image(s) and predict robot actions.",
+  "freeze_action_tokenizer": true,
+  "freeze_vision_encoder": false,
+  "gradient_checkpointing": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "lora_modules_to_save": [
+    "lm_head",
+    "embed_tokens"
+  ],
+  "lora_r": 64,
+  "lora_target_modules": [
+    "q_proj",
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "max_new_tokens": null,
+  "n_action_steps": 16,
+  "n_obs_steps": 1,
+  "name": "bar_boat_pali",
+  "num_action_tokens": null,
+  "pretrained_path": null,
+  "state_prefix_encoding": "discrete_text",
+  "state_prefix_num_bins": 256,
+  "use_lora": false,
+  "vlm_dtype": "bfloat16",
+  "vlm_model_name": "google/paligemma2-3b-mix-224"
+}

BOAT/BOAT_VLA/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2731a2053ebac11974527a9c1830e4452a97474587e6bd0596b0fce83d54671d
+size 6073150736

BOAT/BOAT_VLA/norm_stats.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af2b215b59fbac4e1e7bbece5d95491528f31c95a6c06d017bcf7108b0cc6f5
+size 3957

BOAT/BOAT_VLA/policy_io.json

@@ -0,0 +1,54 @@
+{
+  "obs_features": [
+    {
+      "key": "observation.state",
+      "type": "state",
+      "shape": [
+        14
+      ],
+      "norm_mode": "min_max"
+    },
+    {
+      "key": "observation.images.cam_high",
+      "type": "image",
+      "shape": [
+        3,
+        240,
+        320
+      ],
+      "norm_mode": "mean_std"
+    },
+    {
+      "key": "observation.images.cam_left_wrist",
+      "type": "image",
+      "shape": [
+        3,
+        240,
+        320
+      ],
+      "norm_mode": "mean_std"
+    },
+    {
+      "key": "observation.images.cam_right_wrist",
+      "type": "image",
+      "shape": [
+        3,
+        240,
+        320
+      ],
+      "norm_mode": "mean_std"
+    }
+  ],
+  "action_feature": {
+    "key": "action",
+    "type": "action",
+    "shape": [
+      14
+    ],
+    "norm_mode": "min_max"
+  },
+  "n_obs_steps": 1,
+  "n_action_steps": 32,
+  "fps": 15,
+  "language_key": "task"
+}

BOAT/BOAT_VLA/train_config.yaml

@@ -0,0 +1,134 @@
+policy:
+  pretrained_path: null
+  vlm_model_name: google/paligemma2-3b-mix-224
+  vlm_dtype: bfloat16
+  freeze_vision_encoder: false
+  use_lora: false
+  lora_r: 64
+  lora_alpha: 16
+  lora_target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+  - gate_proj
+  - up_proj
+  - down_proj
+  lora_dropout: 0.05
+  lora_modules_to_save:
+  - lm_head
+  - embed_tokens
+  gradient_checkpointing: true
+  action_token_template: <|action_{i}|>
+  action_embed_init: normal
+  state_prefix_encoding: discrete_text
+  state_prefix_num_bins: 256
+  decode_temperature: null
+  decode_top_k: null
+  default_instruction: Analyze the input image(s) and predict robot actions.
+  action_dim: null
+  n_obs_steps: 1
+  action_horizon: 32
+  n_action_steps: 16
+  action_tokenizer:
+    name: boat
+    kind: tokens
+    pretrained_path: /scratch/xuanweiliu/bimanual_tokenizer/outputs/hopper_boat_robotwin_new/616167.hopper-m-02/train/checkpoints/best/step_300000/pretrained_model
+  freeze_action_tokenizer: true
+  decode_keep_k: null
+  block_ce_weighting: block_mean
+  ce_vocab_scope: full
+  name: bar_boat_pali
+dataset:
+  name: robotwin_new_demo_clean
+  repo_id: local/robotwin_new_demo_clean
+  root: /scratch/xuanweiliu/bimanual_tokenizer/data/lerobot/local/robotwin_new_demo_clean
+  hf_datasets_cache: ${oc.env:HF_DATASETS_CACHE,null}
+  episodes: null
+  exclude_episodes_file: null
+  action_stats_file: null
+  observation_keys:
+  - observation.state
+  - observation.images.cam_high
+  - observation.images.cam_left_wrist
+  - observation.images.cam_right_wrist
+  video_backend: null
+  bypass_force_cache_sync: true
+trainer:
+  max_steps: 30000
+  batch_size: 64
+  global_batch_size: null
+  num_workers: 4
+  prefetch_factor: 4
+  use_episode_aware_sampler: true
+  drop_n_first_frames: 0
+  drop_n_last_frames: 0
+  weight_decay: 1.0e-06
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  lr_scheduler: cosine
+  warmup_steps: 500
+  min_lr_ratio: 0.1
+  use_ema: true
+  ema_decay: 0.995
+  mixed_precision: bf16
+  gradient_accumulation_steps: 1
+  find_unused_parameters: false
+  max_grad_norm: 1.0
+  distributed_strategy: ddp
+  fsdp_min_num_params: 100000000
+  fsdp_reshard_after_forward: true
+  fsdp_cpu_offload: false
+  log_every: 200
+  val_every: 1000
+  checkpoint_every: 1000
+  checkpoint_dir: ${output_dir}/checkpoints
+  local_log_path: ${output_dir}/train_metrics.jsonl
+  val_ratio: 0.05
+  split_seed: ${seed}
+  val_batch_size: null
+  max_val_batches: 100
+  checkpoint_candidates:
+    best:
+      metric_key: val_reconst_loss
+      mode: min
+      k: 2
+    recent:
+      k: 0
+  use_wandb: true
+  wandb_project: praxis-vla
+  wandb_entity: null
+  wandb_init_timeout: null
+  wandb_upload_eval_videos: true
+  wandb_max_eval_videos: 16
+  _trainer_type: policy
+  lrs:
+    vlm: 0.0001
+    vision: 0.0001
+    bridge: 0.0001
+  reconst_every: 1000
+  max_reconst_batches: 100
+  reconst_action_mode: policy
+  sim_eval_every: null
+  sim_eval_start_after_step: 0
+  sim_eval:
+    num_eval_per_task: 20
+    num_parallel_env: 1
+    record_episodes_per_task: 0
+    step_timeout_sec: null
+    rollout_failure_retries: 1
+    policy_kwargs: {}
+name: train_policy
+seed: 42
+device: cuda
+run_name: robotwin_new_bar_boat_pali_30k
+output_dir: /scratch/xuanweiliu/bimanual_tokenizer/outputs/hopper_bar_boat_pali_robotwin_new_30k/616455.hopper-m-02/train
+run_mode: overwrite
+resume_checkpoint: null
+env:
+  type: null
+  task: null
+  task_ids: null
+  python_bin: null