Upload 22 files

.hydra/config.yaml

@@ -0,0 +1,239 @@
+datamodule:
+  transforms:
+    combine_goal_obs: false
+    move_axis: false
+    bytes_to_string: true
+    adjust_type: null
+    add_robot_information: true
+  language_encoders:
+    _target_: flower_vla.agents.lang_encoders.florence_tokens.TokenVLM
+    _recursive_: false
+    model_name: microsoft/Florence-2-large
+  _target_: flower_vla.dataset.datamodule.UhaDataModule
+  _recursive_: false
+  num_workers: ${num_workers}
+  batch_size: ${batch_size}
+  pin_memory: ${pin_memory}
+  drop_last: ${drop_last}
+  datasets:
+    DATA_NAME: ${DATA_NAME}
+    DATA_PATH: gs://gresearch/robotics
+    load_camera_views: ${load_camera_views}
+    dataset_size_limit: ${dataset_size_limit}
+    action_proprio_normalization_type: bounds
+    proprio_obs_key: proprio
+    interleaved_dataset_cfg:
+      shuffle_buffer_size: ${shuffle_buffer_size}
+      balance_weights: true
+      traj_transform_kwargs:
+        goal_relabeling_strategy: ${goal_relabeling_strategy}
+        goal_relabeling_kwargs:
+          goal_relabeling_kwargs:
+            min_bound: 20
+            max_bound: 50
+            frame_diff: 3
+        window_size: ${obs_seq_len}
+        action_horizon: ${act_seq_len}
+        max_action_dim: ${act_dim}
+        max_proprio_dim: ${max_proprio_dim}
+        subsample_length: ${subsample_length}
+        skip_unlabeled: ${skip_unlabeled}
+      frame_transform_kwargs:
+        image_augment_kwargs:
+          primary:
+            random_resized_crop:
+              scale:
+              - 0.8
+              - 1.0
+              ratio:
+              - 0.9
+              - 1.1
+            augment_order:
+            - random_resized_crop
+          secondary:
+            random_resized_crop:
+              scale:
+              - 0.8
+              - 1.0
+              ratio:
+              - 0.9
+              - 1.1
+            augment_order:
+            - random_resized_crop
+          wrist:
+            random_brightness:
+            - 0.1
+            augment_order:
+            - random_brightness
+        resize_size:
+          primary:
+          - 224
+          - 224
+          secondary:
+          - 224
+          - 224
+          wrist:
+          - 224
+          - 224
+        resize_size_future_obs:
+          primary:
+          - 112
+          - 112
+          secondary:
+          - 112
+          - 112
+          wrist:
+          - 112
+          - 112
+        num_parallel_calls: 64
+      traj_transform_threads: 12
+      traj_read_threads: 12
+trainer:
+  agent:
+    _target_: flower_vla.agents.vlp_ddp_wrapper.DDPAgentWrapper
+    _recursive_: false
+    obs_modalities: ${obs_modalities}
+    goal_modalities: ${goal_modalities}
+    img_modalities: ${img_modalities}
+    lang_modalities: ${lang_modalities}
+    target_modality: ${target_modality}
+    agent:
+      _target_: flower_vla.agents.vlp.VLP
+      _recursive_: false
+      obs_modalities: ${obs_modalities}
+      goal_modalities: ${goal_modalities}
+      img_modalities: ${img_modalities}
+      lang_modalities: ${lang_modalities}
+      target_modality: ${target_modality}
+      vlm_path: microsoft/Florence-2-large
+      freeze_florence: false
+      freeze_vision_tower: false
+      freeze_embeddings_only: false
+      use_second_view: ${use_second_view}
+      token_dropout: 0.1
+      cfg_dropout: 0.0
+      second_view_key: ${second_view_key}
+      use_third_view: ${use_third_view}
+      third_view_key: ${third_view_key}
+      multistep: ${act_seq_len}
+      lowdim_obs_dim: ${obs_dim}
+      action_dim: ${act_dim}
+      act_window_size: ${act_seq_len}
+      use_causal_attention: true
+      update_w_bound: ${update_w_bound}
+      use_proprio: ${use_proprio}
+      mp_tokenizer:
+        _target_: flower_vla.agents.vlp_models.bspline_tokenizer.BSpline_Tokenizer
+        _recursive_: false
+        num_dof: ${act_dim}
+        num_basis: ${num_basis}
+        duration: 1.0
+        seq_len: ${act_seq_len}
+        vocab_size: ${vocab_size}
+        degree_p: ${degree_p}
+        gripper_zero_order: ${gripper_zero_order}
+        gripper_dof: ${gripper_dof}
+        gripper_indices: ${gripper_indices}
+        w_max_bound: ${w_max_bound}
+        w_min_bound: ${w_min_bound}
+        init_cond_order: 0
+        end_cond_order: 0
+  _target_: flower_vla.trainers.vlp_trainer.AccelerateTrainer
+  _recursive_: false
+  pre_compute_w_bound: ${pre_compute_w_bound}
+  pre_compute_w_bound_steps: ${pre_compute_w_bound_steps}
+  use_torch_compile: false
+  single_optimizer: true
+  learning_rate_vlm: 2.0e-05
+  use_lr_scheduler: true
+  beta_vlm_1: 0.9
+  beta_vlm_2: 0.999
+  weight_decay:
+    transformer_weight_decay: 0.0
+    vlm_weight_decay: 0.0
+  vlm_lr_scheduler:
+    _target_: flower_vla.utils.lr_schedulers.TriStageLRScheduler
+    init_lr_scale: 1.0
+    final_lr_scale: 1.0
+    phase_ratio: (0.0, 1.0, 0.0)
+    total_steps: 60000
+  eval_every_n_steps: ${eval_every_n_steps}
+  save_every_n_steps: ${save_every_n_steps}
+  max_train_steps: ${max_train_steps}
+  max_eval_steps: ${max_eval_steps}
+  use_ema: false
+  decay: ${decay}
+  rampup_ratio: ${rampup_ratio}
+  update_ema_every_n_steps: ${update_ema_every_n_steps}
+  batch_size: ${batch_size}
+  obs_modalities: ${obs_modalities}
+  goal_modalities: ${goal_modalities}
+  img_modalities: ${img_modalities}
+  lang_modalities: ${lang_modalities}
+  target_modality: ${target_modality}
+DATA_NAME: aloha_right_left_transfer
+wandb:
+  name: Causal_Seq50_basis10_${now:%H-%M-%S}
+  group: Franka_REAL_${now:%Y-%m-%d}
+  project: Aloha_Right_Left_Transfer
+  entity: zhouhongyi
+  mode: null
+log_dir: logs/
+obs_seq_len: 1
+goal_window_size: 1
+seed: 42
+obs_dim: 16
+goal_dim: 512
+update_ema_every_n_steps: 1
+decay: 0.999
+rampup_ratio: 0.001
+goal_relabeling_strategy: null
+subsample_length: null
+skip_unlabeled: true
+load_camera_views:
+- primary
+- secondary
+- wrist
+obs_modalities: observation
+goal_modalities: task
+img_modalities:
+- image_primary
+- image_secondary
+- image_wrist
+lang_modalities:
+- language_instruction
+target_modality: action
+use_proprio: false
+drop_last: true
+pin_memory: true
+num_workers: 0
+use_second_view: true
+second_view_key: image_secondary
+use_third_view: true
+third_view_key: image_wrist
+pre_compute_w_bound: false
+pre_compute_w_bound_steps: 100000
+update_w_bound: true
+w_max_bound: 2.5
+w_min_bound: -2.5
+act_seq_len: 50
+num_basis: 10
+vocab_size: 256
+degree_p: 4
+gripper_zero_order: false
+gripper_dof: 0
+gripper_indices:
+- 6
+- 13
+gradient_accumulation_steps: 1
+find_unused_parameters: true
+static_graph: false
+act_dim: 14
+max_proprio_dim: 16
+max_train_steps: 400000
+max_eval_steps: 100
+eval_every_n_steps: 500
+save_every_n_steps: 5000
+shuffle_buffer_size: 45000
+batch_size: 48
+dataset_size_limit: null

.hydra/hydra.yaml

@@ -0,0 +1,153 @@
+hydra:
+  run:
+    dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+  sweep:
+    dir: ${log_dir}/runs/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra: []
+    task: []
+  job:
+    name: training
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: training_real_aloha.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.1.1
+    cwd: /home/zhou/Codes/vlp_corl25/vla_mp
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/zhou/Codes/vlp_corl25/vla_mp/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    choices:
+      trainer: vlp_trainer
+      trainer/agent: vlp
+      datamodule: oxe_data
+      datamodule/language_encoders: florence
+      datamodule/transforms: oxe_no_remapping
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

checkpoint_10000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:653bbf57b7685bae3d10ab49d6001fae7aa1b8d78b02b38842827770650cc98b
+size 1553613204

checkpoint_10000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8148b9c38f2f48c6fcbdf669b50a6ccfc3cec43e5e5cf491ab85f9d214f3d7d5
+size 3106635726

checkpoint_10000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:42b3c357a7204147eb9aef2bf1fb48e2558f7eb91f8f789e12e1212129544927
+size 15516

checkpoint_10000/random_states_1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc15ff4ae61ef465421488881bb2b706a99bcdd119107e8085d821bcd500672d
+size 15644

checkpoint_10000/random_states_2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca3aa1c255e20b36553933fd17ff3bc34efa60d821ec6f0e8f1207328074ac6d
+size 15516

checkpoint_10000/random_states_3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35252e74f2360b49020c75e9059f15d0d0f50df1e5b8c61a3a1feb1f833719ae
+size 15580

checkpoint_10000/random_states_4.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1daf9d375a56bbf8c00ed47a4d10fb00376eb94abcf55bca5ddda9720f853a59
+size 15580

checkpoint_10000/random_states_5.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:503febac82261fdc3a4c9c0d6e22eb15c2c6904e8cf79d1a04b1e5943de446f7
+size 15580

checkpoint_10000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:673eaceedf150be0d5d38f7bb933bc59cae3b494bc3de4a0e4368aad8c5c42ce
+size 1256

checkpoint_5000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b75418a592d2810a041d3e9c3c44611eca2a721f638bbdb98afee8e12ffafe0
+size 1553613204

checkpoint_5000/optimizer.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9ec7174642ffd8ee129f31a56672b0f806ea564700d7424385d1c8503c9baeb
+size 3106635726

checkpoint_5000/random_states_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b741376a9efe75bb7a31f8b47b629ef0ab7c896eaf75e520b632b26d9b51ea26
+size 15516

checkpoint_5000/random_states_1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4637642b1a28e7f703a17c5a1fededcd5359e30aea7f9e2084e346b1336e9dc8
+size 15644

checkpoint_5000/random_states_2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b35b5d09ce5bcd2f3da879358da7ed858990fd3d0a5d91b84f8ca2c14bcd1610
+size 15516

checkpoint_5000/random_states_3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50f7ffbf0f39828705fa2e2786ecd96e7b1cd59d1289477b63f3dffb781665a4
+size 15580

checkpoint_5000/random_states_4.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d693b8f8391834f2cfc0d1c2730a271c5fd850911e61b5f84cee64eaeccc34e
+size 15580

checkpoint_5000/random_states_5.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f8bc22e86d981013cb5990c18b070b82a8ef3d50d189c32cbfdda3355674d1
+size 15580

checkpoint_5000/scheduler.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f32c21440846376ff23ec47b65055a106bb3e55ff0f56b37e8b1a8c8def4fe6e
+size 1256

dataset_statistics_7ff1699aae8e76d63d7a0b4b4b234a73b16a7df6574a1da4bd769893edfa7d26.json

@@ -0,0 +1 @@
+{"action": {"mean": [-0.09980551898479462, -0.7855625748634338, 1.2624194622039795, -0.03944704681634903, -0.34413042664527893, 0.33048224449157715, 0.40334537625312805, 0.07670194655656815, -0.0887017771601677, 0.7883964776992798, 0.06364128738641739, -0.5776045918464661, 0.21130876243114471, 0.2664165794849396], "std": [0.08050194382667542, 0.20292383432388306, 0.10817132145166397, 0.05838508903980255, 0.2088155299425125, 0.3349650502204895, 0.39498281478881836, 0.14862872660160065, 0.6699551939964294, 0.459709107875824, 0.26898688077926636, 0.39960718154907227, 0.37265732884407043, 0.5169549584388733], "max": [0.10737866163253784, -0.09357283264398575, 1.5033012628555298, 0.058291271328926086, 0.6258642077445984, 1.306951642036438, 1.1351457834243774, 0.6519418358802795, 0.8130098581314087, 1.526310920715332, 1.1919031143188477, 1.366776943206787, 1.2563302516937256, 1.320757508277893], "min": [-0.3436117172241211, -1.1351457834243774, 0.6550098061561584, -0.34821364283561707, -0.7271069288253784, -0.08283496648073196, -0.6181942820549011, -0.48166999220848083, -1.667437195777893, -0.7148350477218628, -0.9694758653640747, -1.2041749954223633, -0.6550098061561584, -0.7623884677886963], "p99": [0.06289321184158325, -0.28685441613197327, 1.41586434841156, 0.0475534051656723, 0.3765309420228103, 1.1738481092453052, 1.0983302593231201, 0.5307573676109314, 0.6718835830688477, 1.5002332925796509, 0.7761942744255066, 0.5951845645904541, 1.0986830759048511, 1.2486604452133179], "p01": [-0.3206019997596741, -1.118272066116333, 0.8264628481864928, -0.25039167881011964, -0.6795535087585449, -0.061359234154224396, -0.5767768025398254, -0.3497476279735565, -1.6260197162628174, -0.04295146465301514, -0.52001953125, -1.0848772096633912, -0.49087387323379517, -0.7409127354621887]}, "num_transitions": 35178, "num_trajectories": 52, "proprio": {"mean": [-0.10121026635169983, -0.7844730615615845, 1.246608018875122, -0.0409359484910965, -0.35148757696151733, 0.32990169525146484, 0.22357597947120667, 0.07812093198299408, -0.12076649069786072, 0.772532045841217, 0.06426404416561127, -0.583050549030304, 0.21019193530082703, 0.0971749871969223], "std": [0.08043123036623001, 0.2005949318408966, 0.10898596048355103, 0.06003936007618904, 0.21178223192691803, 0.3343738913536072, 0.5264418125152588, 0.14978328347206116, 0.6600867509841919, 0.4669239819049835, 0.26905491948127747, 0.4049088656902313, 0.3731350600719452, 0.6083250045776367], "max": [0.11044661700725555, -0.09970875084400177, 1.494097352027893, 0.05982525274157524, 0.630466103553772, 1.3023496866226196, 1.1338573694229126, 0.6550098061561584, 0.780796229839325, 1.512505054473877, 1.1919031143188477, 1.3683109283447266, 1.2547962665557861, 1.3202457427978516], "min": [-0.34514567255973816, -1.1290098428726196, 0.6243302226066589, -0.3528155982494354, -0.7424467206001282, -0.08590292930603027, -0.7455589771270752, -0.48166999220848083, -1.656699299812317, -0.7455146908760071, -0.9694758653640747, -1.2164467573165894, -0.6580777764320374, -0.8180433511734009], "p99": [0.061359234154224396, -0.27458256483078003, 1.3974565267562866, 0.052155349403619766, 0.3891555869579463, 1.1726669406890968, 1.097615122795105, 0.5322913527488708, 0.6366020441055298, 1.4894953966140747, 0.7731263637542725, 0.5921165943145752, 1.0956151056289722, 1.2477613687515259], "p01": [-0.3129321038722992, -1.1090681552886963, 0.8222137093544006, -0.25310683250427246, -0.6887574195861816, -0.06289321184158325, -0.6627196669578552, -0.34821364283561707, -1.6183497905731201, -0.17794176936149597, -0.52001953125, -1.0925471353530884, -0.49087387323379517, -0.7921559810638428]}}