Upload folder using huggingface_hub

checkpoint-100.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:339496dfee3637ce8f2f48d3ba979a246afaaaf280ef9e80d8e52436b2f66874
+size 1086629370

checkpoint-prism-100.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:974bd55dd7e46c1525a8962ac5979d0d54b87bf65433eeaba1f9eab6b3c6eb42
+size 1086625786

events.out.tfevents.1768103623.xzhang-vla-h100-8gpu-group-worker-r8mgv.2454659.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:781c07ca7503f877231ac7b5642603a90317633b9e3b9d0279ff0c7dae33fce5
+size 29089182

log.txt

@@ -0,0 +1,102 @@
+{"train_lr": 0.00012471482889733843, "train_loss": 0.2614591849792196, "epoch": 0}
+{"train_lr": 0.00037471482889734324, "train_loss": 0.12663059274411048, "epoch": 1}
+{"train_lr": 0.0004999895472132218, "train_loss": 0.09677751909757296, "epoch": 2}
+{"train_lr": 0.000499926685471894, "train_loss": 0.08038228015521605, "epoch": 3}
+{"train_lr": 0.0004998009060240483, "train_loss": 0.0731114828145351, "epoch": 4}
+{"train_lr": 0.000499612240533964, "train_loss": 0.06743056494452257, "epoch": 5}
+{"train_lr": 0.000499360736497216, "train_loss": 0.062372813043662635, "epoch": 6}
+{"train_lr": 0.0004990464572285422, "train_loss": 0.057157741970853604, "epoch": 7}
+{"train_lr": 0.0004986694818461203, "train_loss": 0.05048772302172908, "epoch": 8}
+{"train_lr": 0.0004982299052514732, "train_loss": 0.04541368942849423, "epoch": 9}
+{"train_lr": 0.0004977278381056517, "train_loss": 0.04234473540966431, "epoch": 10}
+{"train_lr": 0.00049716340680136, "train_loss": 0.037231520822470006, "epoch": 11}
+{"train_lr": 0.0004965367534311785, "train_loss": 0.035030936513365224, "epoch": 12}
+{"train_lr": 0.0004958480357517376, "train_loss": 0.0332998373289504, "epoch": 13}
+{"train_lr": 0.000495097427144019, "train_loss": 0.03114408211555941, "epoch": 14}
+{"train_lr": 0.0004942851165697412, "train_loss": 0.029543039859963063, "epoch": 15}
+{"train_lr": 0.0004934113085237665, "train_loss": 0.027821352950964804, "epoch": 16}
+{"train_lr": 0.0004924762229825477, "train_loss": 0.026566345540358697, "epoch": 17}
+{"train_lr": 0.0004914800953488991, "train_loss": 0.025293596562869253, "epoch": 18}
+{"train_lr": 0.0004904231763926251, "train_loss": 0.024442589124850575, "epoch": 19}
+{"train_lr": 0.0004893057321874037, "train_loss": 0.023873297852417985, "epoch": 20}
+{"train_lr": 0.0004881280440438942, "train_loss": 0.023207714826190853, "epoch": 21}
+{"train_lr": 0.0004868904084387021, "train_loss": 0.022484266709176887, "epoch": 22}
+{"train_lr": 0.00048559313694003015, "train_loss": 0.021617892973378173, "epoch": 23}
+{"train_lr": 0.0004842365561290255, "train_loss": 0.02093767276658772, "epoch": 24}
+{"train_lr": 0.0004828210075176642, "train_loss": 0.020491828908249667, "epoch": 25}
+{"train_lr": 0.00048134684746271964, "train_loss": 0.02013684416542134, "epoch": 26}
+{"train_lr": 0.0004798144470761096, "train_loss": 0.019881077560068537, "epoch": 27}
+{"train_lr": 0.00047822419213142993, "train_loss": 0.019623963291802844, "epoch": 28}
+{"train_lr": 0.0004765764829668789, "train_loss": 0.019323614682106322, "epoch": 29}
+{"train_lr": 0.00047487173438442494, "train_loss": 0.018956802155337377, "epoch": 30}
+{"train_lr": 0.0004731103755453882, "train_loss": 0.018540869315841833, "epoch": 31}
+{"train_lr": 0.00047129284986243867, "train_loss": 0.01824583707757988, "epoch": 32}
+{"train_lr": 0.00046941961488795723, "train_loss": 0.017965269687774985, "epoch": 33}
+{"train_lr": 0.0004674911421988518, "train_loss": 0.017682267398539817, "epoch": 34}
+{"train_lr": 0.00046550791727776485, "train_loss": 0.017457642718678616, "epoch": 35}
+{"train_lr": 0.00046347043939099045, "train_loss": 0.017055699007582102, "epoch": 36}
+{"train_lr": 0.0004613792214626758, "train_loss": 0.016706142773525966, "epoch": 37}
+{"train_lr": 0.0004592347899457634, "train_loss": 0.016423434982653494, "epoch": 38}
+{"train_lr": 0.00045703768468944076, "train_loss": 0.016194938451552784, "epoch": 39}
+{"train_lr": 0.000454788458803156, "train_loss": 0.015942464264839206, "epoch": 40}
+{"train_lr": 0.0004524876785175438, "train_loss": 0.01568605488005091, "epoch": 41}
+{"train_lr": 0.0004501359230417095, "train_loss": 0.015525980918374295, "epoch": 42}
+{"train_lr": 0.0004477337844175323, "train_loss": 0.015395837523815126, "epoch": 43}
+{"train_lr": 0.00044528186737054004, "train_loss": 0.01528227873121746, "epoch": 44}
+{"train_lr": 0.0004427807891577693, "train_loss": 0.015230634887021092, "epoch": 45}
+{"train_lr": 0.0004402311794122911, "train_loss": 0.015006808040073998, "epoch": 46}
+{"train_lr": 0.0004376336799846814, "train_loss": 0.014920323177731239, "epoch": 47}
+{"train_lr": 0.0004349889447815459, "train_loss": 0.01475530250378647, "epoch": 48}
+{"train_lr": 0.000432297639600863, "train_loss": 0.014665135457277823, "epoch": 49}
+{"train_lr": 0.0004295604419642602, "train_loss": 0.014575754177390099, "epoch": 50}
+{"train_lr": 0.0004267780409466762, "train_loss": 0.014445972053197783, "epoch": 51}
+{"train_lr": 0.0004239511370026099, "train_loss": 0.014355132175651963, "epoch": 52}
+{"train_lr": 0.00042108044179007485, "train_loss": 0.014255816408923469, "epoch": 53}
+{"train_lr": 0.000418166677991226, "train_loss": 0.0142102003218553, "epoch": 54}
+{"train_lr": 0.00041521057913047063, "train_loss": 0.014086610997881839, "epoch": 55}
+{"train_lr": 0.00041221288938992755, "train_loss": 0.013995079127425406, "epoch": 56}
+{"train_lr": 0.00040917436342199, "train_loss": 0.013965443669439439, "epoch": 57}
+{"train_lr": 0.0004060957661592657, "train_loss": 0.013818377612889904, "epoch": 58}
+{"train_lr": 0.00040297787262215975, "train_loss": 0.01365872271601093, "epoch": 59}
+{"train_lr": 0.00039982146772372105, "train_loss": 0.013509774950032827, "epoch": 60}
+{"train_lr": 0.00039662734607200155, "train_loss": 0.01334795588471647, "epoch": 61}
+{"train_lr": 0.00039339631177000956, "train_loss": 0.013251997352561811, "epoch": 62}
+{"train_lr": 0.0003901291782134003, "train_loss": 0.013107434170924396, "epoch": 63}
+{"train_lr": 0.0003868267678855275, "train_loss": 0.013025237130658071, "epoch": 64}
+{"train_lr": 0.00038348991215056936, "train_loss": 0.012972700235282492, "epoch": 65}
+{"train_lr": 0.0003801194510440964, "train_loss": 0.012815852962587446, "epoch": 66}
+{"train_lr": 0.0003767162330616758, "train_loss": 0.012665181592976171, "epoch": 67}
+{"train_lr": 0.00037328111494522016, "train_loss": 0.012596371038316682, "epoch": 68}
+{"train_lr": 0.0003698149614673644, "train_loss": 0.012520412869902837, "epoch": 69}
+{"train_lr": 0.0003663186452136829, "train_loss": 0.012444866185511211, "epoch": 70}
+{"train_lr": 0.00036279304636311656, "train_loss": 0.012373207830177698, "epoch": 71}
+{"train_lr": 0.00035923905246633983, "train_loss": 0.01229620802455702, "epoch": 72}
+{"train_lr": 0.00035565755822226404, "train_loss": 0.012195132931830723, "epoch": 73}
+{"train_lr": 0.00035204946525293866, "train_loss": 0.01214081704138934, "epoch": 74}
+{"train_lr": 0.0003484156818764563, "train_loss": 0.01210959582919779, "epoch": 75}
+{"train_lr": 0.0003447571228784029, "train_loss": 0.012004285899310518, "epoch": 76}
+{"train_lr": 0.00034107470928137053, "train_loss": 0.011985535857789113, "epoch": 77}
+{"train_lr": 0.00033736936811332684, "train_loss": 0.011835613351416183, "epoch": 78}
+{"train_lr": 0.0003336420321741082, "train_loss": 0.011715506594327098, "epoch": 79}
+{"train_lr": 0.00032989363980056327, "train_loss": 0.011599909779593569, "epoch": 80}
+{"train_lr": 0.00032612513463046143, "train_loss": 0.011584922045384296, "epoch": 81}
+{"train_lr": 0.0003223374653648161, "train_loss": 0.011511126754937354, "epoch": 82}
+{"train_lr": 0.00031853158552909883, "train_loss": 0.011398970488270306, "epoch": 83}
+{"train_lr": 0.0003147084532332163, "train_loss": 0.01135708225666131, "epoch": 84}
+{"train_lr": 0.00031086903093028987, "train_loss": 0.011319569532392978, "epoch": 85}
+{"train_lr": 0.0003070142851743234, "train_loss": 0.011239001153151153, "epoch": 86}
+{"train_lr": 0.0003031451863769697, "train_loss": 0.01114756022767649, "epoch": 87}
+{"train_lr": 0.00029926270856316036, "train_loss": 0.01112312617122564, "epoch": 88}
+{"train_lr": 0.0002953678291258921, "train_loss": 0.011084513694146194, "epoch": 89}
+{"train_lr": 0.0002914615285802815, "train_loss": 0.011036451935069636, "epoch": 90}
+{"train_lr": 0.00028754479031656933, "train_loss": 0.010965720450008968, "epoch": 91}
+{"train_lr": 0.00028361860035266853, "train_loss": 0.010917112885955756, "epoch": 92}
+{"train_lr": 0.00027968394708591634, "train_loss": 0.010847472263847127, "epoch": 93}
+{"train_lr": 0.00027574182104422943, "train_loss": 0.010813182947919048, "epoch": 94}
+{"train_lr": 0.0002717932146367527, "train_loss": 0.010793620029308554, "epoch": 95}
+{"train_lr": 0.000267839121904037, "train_loss": 0.01073198472624627, "epoch": 96}
+{"train_lr": 0.0002638805382677826, "train_loss": 0.010701668241681856, "epoch": 97}
+{"train_lr": 0.00025991846028024956, "train_loss": 0.010622225713862121, "epoch": 98}
+{"train_lr": 0.00025595388537337917, "train_loss": 0.010560437409417924, "epoch": 99}
+{"train_lr": 0.00025198781160771063, "train_loss": 0.01054605341583717, "epoch": 100}
+{"train_lr": 0.0002480212374210882, "train_loss": 0.01048966632852731, "epoch": 101}

results_s0_evals20_tt_window_obs_checkpoint-100.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.2,MemFruitInSinkRightFar
+20,0.05,MemFruitInSinkLeftFar
+20,0.35,MemPutKBreadInMicrowave
+20,0.15,MemWashAndReturnRight
+20,0.5,MemWashAndReturnLeft
+20,0.75,MemHeatPot
+20,0.2,MemHeatPotMultiple
+20,0.05,MemPutKBowlInCabinet
+20,0.4,MemRetrieveOilsFromCounterLL
+20,0.6,MemRetrieveOilsFromCounterLR
+20,0.5,MemRetrieveOilsFromCounterRL
+20,0.05,MemRetrieveOilsFromCounterRR
+20,0.2,MemWashAndReturnSameLocation

results_s0_evals20_tt_window_obs_checkpoint-50.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.25,MemFruitInSinkRightFar
+20,0.15,MemFruitInSinkLeftFar
+20,0.3,MemPutKBreadInMicrowave
+20,0.2,MemWashAndReturnRight
+20,0.55,MemWashAndReturnLeft
+20,0.6,MemHeatPot
+20,0.2,MemHeatPotMultiple
+20,0.55,MemPutKBowlInCabinet
+20,0.15,MemRetrieveOilsFromCounterLL
+20,0.35,MemRetrieveOilsFromCounterLR
+20,0.4,MemRetrieveOilsFromCounterRL
+20,0.2,MemRetrieveOilsFromCounterRR
+20,0.05,MemWashAndReturnSameLocation

results_s0_evals20_tt_window_obs_checkpoint-60.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.25,MemFruitInSinkRightFar
+20,0.15,MemFruitInSinkLeftFar
+20,0.2,MemPutKBreadInMicrowave
+20,0.4,MemWashAndReturnRight
+20,0.4,MemWashAndReturnLeft
+20,0.55,MemHeatPot
+20,0.1,MemHeatPotMultiple
+20,0.35,MemPutKBowlInCabinet
+20,0.25,MemRetrieveOilsFromCounterLL
+20,0.5,MemRetrieveOilsFromCounterLR
+20,0.45,MemRetrieveOilsFromCounterRL
+20,0.25,MemRetrieveOilsFromCounterRR
+20,0.1,MemWashAndReturnSameLocation

results_s0_evals20_tt_window_obs_checkpoint-70.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.2,MemFruitInSinkRightFar
+20,0.4,MemFruitInSinkLeftFar
+20,0.2,MemPutKBreadInMicrowave
+20,0.35,MemWashAndReturnRight
+20,0.4,MemWashAndReturnLeft
+20,0.75,MemHeatPot
+20,0.25,MemHeatPotMultiple
+20,0.3,MemPutKBowlInCabinet
+20,0.2,MemRetrieveOilsFromCounterLL
+20,0.4,MemRetrieveOilsFromCounterLR
+20,0.6,MemRetrieveOilsFromCounterRL
+20,0.15,MemRetrieveOilsFromCounterRR
+20,0.0,MemWashAndReturnSameLocation

results_s0_evals20_tt_window_obs_checkpoint-80.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.35,MemFruitInSinkRightFar
+20,0.35,MemFruitInSinkLeftFar
+20,0.25,MemPutKBreadInMicrowave
+20,0.2,MemWashAndReturnRight
+20,0.35,MemWashAndReturnLeft
+20,0.75,MemHeatPot
+20,0.1,MemHeatPotMultiple
+20,0.3,MemPutKBowlInCabinet
+20,0.4,MemRetrieveOilsFromCounterLL
+20,0.45,MemRetrieveOilsFromCounterLR
+20,0.65,MemRetrieveOilsFromCounterRL
+20,0.15,MemRetrieveOilsFromCounterRR
+20,0.05,MemWashAndReturnSameLocation

results_s0_evals20_tt_window_obs_checkpoint-90.csv

@@ -0,0 +1,14 @@
+n_eval,success_rate,task_name
+20,0.3,MemFruitInSinkRightFar
+20,0.15,MemFruitInSinkLeftFar
+20,0.2,MemPutKBreadInMicrowave
+20,0.35,MemWashAndReturnRight
+20,0.55,MemWashAndReturnLeft
+20,0.6,MemHeatPot
+20,0.1,MemHeatPotMultiple
+20,0.4,MemPutKBowlInCabinet
+20,0.25,MemRetrieveOilsFromCounterLL
+20,0.45,MemRetrieveOilsFromCounterLR
+20,0.55,MemRetrieveOilsFromCounterRL
+20,0.2,MemRetrieveOilsFromCounterRR
+20,0.2,MemWashAndReturnSameLocation

run-prism.yaml

@@ -0,0 +1,137 @@
+dataset_cfg:
+  action_noise: 0.0
+  dataset_json:
+  - /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/task_robocasa_mem_eight.json
+  dataset_val_json:
+  - ''
+  load_in_mem: false
+  n_examples_only: -1
+  non_overlapping:
+  - 2
+  num_repeat_traj:
+  - 256
+  proprio_noise: 0.005
+  sample_each_state: false
+  task_name: ''
+  task_names: null
+  train_on_exploration: false
+  use_dali: true
+dataset_json_contents:
+- action_keys:
+  - actions
+  dataset_path:
+  - memory/MemPutKBreadInMicrowave/2025-07-25-10-40-55/demo_im128_notp.hdf5
+  - memory/MemPutKBreadInMicrowave/2025-07-25-13-25-05/demo_im128_notp.hdf5
+  - memory/MemPutKBreadInMicrowave/2025-07-25-14-50-59/demo_im128_notp.hdf5
+  - memory/MemFruitInSinkLeftFar/2025-07-13-15-44-32/demo_im128_notp.hdf5
+  - memory/MemFruitInSinkRightFar/2025-07-15-15-07-57/demo_im128_notp.hdf5
+  - memory/MemHeatPot/2025-07-24-22-26-20/demo_im128_notp.hdf5
+  - memory/MemHeatPot/2025-07-25-15-56-53/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnLeft/2025-07-25-00-12-14/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnRight/2025-07-25-00-45-53/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-10-55/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-35-24/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-50-23/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-21-08-42/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-21-35-46/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed0_2025-09-19-23-10-47_6demo/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed1_2025-09-19-23-32-29_20demo/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed2_2025-09-20-00-42-49_30demo/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterLL/2025-09-20-22-49-27/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterLR/2025-09-20-23-25-19/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterRL/2025-09-20-23-06-15/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterRR/2025-09-20-23-14-26/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-21-52-31/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-22-11-00/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-22-25-19/demo_im128_notp.hdf5
+  group_method: dinov2
+  image_keys:
+  - obs/robot0_agentview_center_image
+  - obs/robot0_eye_in_hand_image
+  low_dim_keys: []
+  max_traj_len: 2048
+  proprio_keys:
+  - obs/robot0_joint_pos_cos
+  - obs/robot0_gripper_qpos
+device: cuda
+dist_on_itp: false
+dist_url: env://
+load_config: null
+logging_cfg:
+  log_dir: /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+  log_name: exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+  output_dir: /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+model_cfg:
+  policy_cfg:
+    attn_drop: 0.0
+    attn_scale_factor: null
+    block_attn_ind: null
+    block_pattern_start_offset: 0
+    compress_full_attn: true
+    compressor_latent_len: 1
+    decoder_pred_head: mlp
+    full_attn_inds: null
+    gate_full_attn_layers: true
+    gating_type: sigmoid_g1
+    lora_rank: 4
+    model_version: v2
+    phase: pretrain
+    pool_block_inds: null
+    position_language_embedding: start
+    pretrained_path: null
+    remove_action: false
+    remove_proprio: false
+    scratch_llama_config: config/model_config/libero_1x.json
+    tt_window_obs: false
+    use_block_attention: true
+    use_linear_attention: false
+    use_lstm: false
+    use_mamba: false
+    use_transformer_xl: false
+    xl_chunk_factor: 32
+    xl_mem_len: 32
+  vision_encoder_cfg:
+    freeze_all: false
+    vision_encoder: /storage/nfs/rshah/datasets/robocasa/datasets/crossmae_rtx/cross-mae-rtx-vitb.pth
+    vision_lora: false
+    vision_lora_rank: 8
+    vision_nonpretrained: false
+    vision_unfreeze_all: false
+    vision_unfreeze_last_n: 0
+optimizer_cfg:
+  blr: 0.001
+  lr: 0.0005
+  min_lr: 0.0
+  warmup_epochs: 2.0
+  weight_decay: 0.01
+shared_cfg:
+  batch_size: 2
+  block_finetune: ''
+  downsample_obs: 8
+  enable_gradient_checkpointing: false
+  gripper_img: true
+  has_base_action: true
+  is_bimanual: false
+  k_ptp: 0
+  num_cameras: 2
+  num_pred_steps: 32
+  pad_to_max_length: true
+  resume: null
+  save_every: 10
+  seed: 2
+  seq_length: 2048
+  split_epoch: 1
+  start_epoch: 0
+  use_language_conditioning: true
+train: true
+trainer_cfg:
+  accum_iter: 32
+  break_after_n_epochs: 101
+  compile_model: true
+  epochs: 200
+  num_workers: 72
+  pin_memory: true
+  val_every: 10
+  wandb_project: icrt_reruns
+  wandb_watch: true
+world_size: 1

run.yaml

@@ -0,0 +1,238 @@
+!!python/object:icrt.util.args.ExperimentConfig
+dataset_cfg: !!python/object:icrt.util.args.DatasetConfig
+  action_noise: 0.0
+  action_traj_noise: false
+  dataset_fraction: 1.0
+  dataset_json:
+  - /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/task_robocasa_mem_eight.json
+  dataset_val_json:
+  - ''
+  goal_conditioned: false
+  load_in_mem: false
+  low_dim_noise: 0.005
+  min_subsample_length: 64
+  n_examples_only: -1
+  non_overlapping:
+  - 2
+  num_repeat_traj:
+  - 256
+  num_weighted_steps: 30
+  only_k: 10
+  proprio_noise: 0.005
+  random_patch_masking: false
+  rebalance_indices: false
+  rebalance_tasks: true
+  sample_each_state: false
+  shuffle_repeat_traj: true
+  shuffle_rnd_prob: 0.0
+  sim_score_th: -1.0
+  skip_step: false
+  sort_by_closeness: false
+  sort_by_lang: true
+  start_from_beginning: false
+  subsample_traj: false
+  task_barrier: true
+  task_name: ''
+  task_names: null
+  toy_max_distractors: 1
+  toy_max_tasks: 5
+  toy_task_id: 1
+  toy_train_dataset_size: 1000000
+  toy_train_seed: 42
+  toy_val_dataset_size: 500000
+  toy_val_seed: 84
+  train_on_exploration: false
+  trim_num: -1
+  trim_set: ''
+  use_dali: true
+  val_only_k: 5
+  video_conditioned: false
+  vision_aug: true
+  weight_by_dataset:
+  - 1
+dataset_json_contents:
+- action_keys:
+  - actions
+  dataset_path:
+  - memory/MemPutKBreadInMicrowave/2025-07-25-10-40-55/demo_im128_notp.hdf5
+  - memory/MemPutKBreadInMicrowave/2025-07-25-13-25-05/demo_im128_notp.hdf5
+  - memory/MemPutKBreadInMicrowave/2025-07-25-14-50-59/demo_im128_notp.hdf5
+  - memory/MemFruitInSinkLeftFar/2025-07-13-15-44-32/demo_im128_notp.hdf5
+  - memory/MemFruitInSinkRightFar/2025-07-15-15-07-57/demo_im128_notp.hdf5
+  - memory/MemHeatPot/2025-07-24-22-26-20/demo_im128_notp.hdf5
+  - memory/MemHeatPot/2025-07-25-15-56-53/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnLeft/2025-07-25-00-12-14/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnRight/2025-07-25-00-45-53/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-10-55/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-35-24/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-20-50-23/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-21-08-42/demo_im128_notp.hdf5
+  - memory/MemHeatPotMultiple/2025-09-20-21-35-46/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed0_2025-09-19-23-10-47_6demo/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed1_2025-09-19-23-32-29_20demo/demo_im128_notp.hdf5
+  - memory/MemPutKBowlInCabinet/seed2_2025-09-20-00-42-49_30demo/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterLL/2025-09-20-22-49-27/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterLR/2025-09-20-23-25-19/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterRL/2025-09-20-23-06-15/demo_im128_notp.hdf5
+  - memory/MemRetrieveOilsFromCounterRR/2025-09-20-23-14-26/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-21-52-31/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-22-11-00/demo_im128_notp.hdf5
+  - memory/MemWashAndReturnSameLocation/2025-09-20-22-25-19/demo_im128_notp.hdf5
+  group_method: dinov2
+  image_keys:
+  - obs/robot0_agentview_center_image
+  - obs/robot0_eye_in_hand_image
+  low_dim_keys: []
+  max_traj_len: 2048
+  proprio_keys:
+  - obs/robot0_joint_pos_cos
+  - obs/robot0_gripper_qpos
+device: cuda
+dist_on_itp: false
+dist_url: env://
+load_config: null
+local_rank: -1
+logging_cfg: !!python/object:icrt.util.args.LoggingConfig
+  log_dir: /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+  log_name: exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+  output_dir: /storage/nfs/xzhang/experiments/nopool_exps/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress/exp_ds8_libero_1x_task_robocasa_mem_eight_GBS256_cfgtask_robocasa_mem_eight_sl2048_s2_crossmae_block_sigmoid_g1_nopool_compress
+model_cfg: !!python/object:icrt.util.args.ModelConfig
+  policy_cfg: !!python/object:icrt.util.args.PolicyConfig
+    adapter_mlp_ratio: 4.0
+    adapter_num_heads: 8
+    add_sink_token_to_full_attn: false
+    attn_drop: 0.0
+    attn_latent_len: 1
+    attn_scale_factor: null
+    block_attn_ind: null
+    block_pattern_start_offset: 0
+    compress_full_attn: true
+    compress_include_prev_action: false
+    compressor_latent_len: 1
+    decoder_cfg_path: config/model_config/action_head.json
+    decoder_pred_head: mlp
+    full_attn_inds: null
+    gate_full_attn_layers: true
+    gating_type: sigmoid_g1
+    gr00t_add_dit_lora: false
+    gr00t_add_full_attn: true
+    gr00t_add_lora_to_vision_projector: false
+    gr00t_compute_dtype: bfloat16
+    gr00t_full_attn_idx:
+    - 3
+    - 7
+    - 11
+    - 15
+    gr00t_pretrained_model: true
+    gr00t_tune_diffusion_model: true
+    gr00t_tune_llm: false
+    gr00t_tune_projector: true
+    gr00t_tune_vision_projector: false
+    gr00t_tune_visual: false
+    hist_attn_latent_len: 1
+    hist_blind_gating: false
+    hist_feature_gating: false
+    interpolation_factor: null
+    llama_ckpt_dir: /home/rutavms/.llama/checkpoints/Llama3.2-1B-Instruct
+    load_llama: true
+    lora_layer_idxs: null
+    lora_rank: 4
+    loss_w_action: 1.0
+    model_version: v2
+    multikv_attn_pool: false
+    no_prompt_loss: true
+    num_inference_diffusion_steps: null
+    num_train_diffusion_steps: null
+    phase: pretrain
+    pool_block_inds: null
+    position_language_embedding: start
+    pretrained_path: null
+    prop_act_norm: null
+    remove_action: false
+    remove_proprio: false
+    scale_loss: 1.0
+    scratch_llama_config: config/model_config/libero_1x.json
+    separate_camera_adapter: true
+    skip_obs_pooling: true
+    step_weight: 1.0
+    tt_window_obs: false
+    use_block_attention: true
+    use_linear_attention: false
+    use_lstm: false
+    use_mamba: false
+    use_reshape_block_attn: false
+    use_transformer_xl: false
+    use_yarn: null
+    vision_projection_layer: false
+    xl_chunk_factor: 32
+    xl_mem_len: 32
+  vision_encoder_cfg: !!python/object:icrt.util.args.VisionEncoderConfig
+    bbox_vision_encoder_cfg: config/model_config/bbox_vit_128_16.json
+    freeze_all: false
+    vision_encoder: /storage/nfs/rshah/datasets/robocasa/datasets/crossmae_rtx/cross-mae-rtx-vitb.pth
+    vision_lora: false
+    vision_lora_rank: 8
+    vision_nonpretrained: false
+    vision_unfreeze_all: false
+    vision_unfreeze_last_n: 0
+optimizer_cfg: !!python/object:icrt.util.args.OptimizerConfig
+  blr: 0.001
+  lr: 0.0005
+  min_lr: 0.0
+  warmup_epochs: 2.0
+  weight_decay: 0.01
+shared_cfg: !!python/object:icrt.util.args.SharedConfig
+  batch_size: 2
+  bbox_noise: 0.015625
+  block_finetune: ''
+  downsample_obs: 8
+  drop_bbox_prob: 0.0
+  dropping_prompt_rnd: false
+  enable_gradient_checkpointing: false
+  gr00t_embodiment_str: single_panda_omron
+  gr00t_metadata_path: metadata.json
+  gr00t_model_path: nvidia/GR00T-N1.5-3B
+  gripper_img: true
+  has_base_action: true
+  hist_seq_length: 2048
+  is_bimanual: false
+  k_ptp: 0
+  max_bbox_per_img: 15
+  no_img: false
+  no_img_low_dim: 42
+  normalize_io: false
+  num_cameras: 2
+  num_pred_steps: 32
+  num_stages: 1
+  only_first_obs: false
+  pad_to_max_length: true
+  resume: null
+  rot_6d: false
+  rot_euler: false
+  save_every: 10
+  seed: 2
+  seq_length: 2048
+  split_epoch: 1
+  start_epoch: 0
+  state_center_coords: false
+  tokenizer_path: physical-intelligence/fast
+  use_bboxes: false
+  use_delta_action: false
+  use_fast_tokenizer: false
+  use_gr00t: false
+  use_history: false
+  use_language_conditioning: true
+  use_toy_vision_dataset: false
+  use_vqvae_tokenizer: false
+train: true
+trainer_cfg: !!python/object:icrt.util.args.TrainerConfig
+  accum_iter: 32
+  break_after_n_epochs: 101
+  compile_model: true
+  epochs: 200
+  num_workers: 72
+  pin_memory: true
+  val_every: 10
+  wandb_project: icrt_reruns
+  wandb_watch: true
+world_size: 1

run_id.txt

@@ -0,0 +1 @@
+zmzny5vb

task_robocasa_mem_eight.json

@@ -0,0 +1,42 @@
+{
+    "dataset_path": [
+        "memory/MemPutKBreadInMicrowave/2025-07-25-10-40-55/demo_im128_notp.hdf5",
+        "memory/MemPutKBreadInMicrowave/2025-07-25-13-25-05/demo_im128_notp.hdf5",
+        "memory/MemPutKBreadInMicrowave/2025-07-25-14-50-59/demo_im128_notp.hdf5",
+        "memory/MemFruitInSinkLeftFar/2025-07-13-15-44-32/demo_im128_notp.hdf5",
+        "memory/MemFruitInSinkRightFar/2025-07-15-15-07-57/demo_im128_notp.hdf5",
+        "memory/MemHeatPot/2025-07-24-22-26-20/demo_im128_notp.hdf5",
+        "memory/MemHeatPot/2025-07-25-15-56-53/demo_im128_notp.hdf5",
+        "memory/MemWashAndReturnLeft/2025-07-25-00-12-14/demo_im128_notp.hdf5",
+        "memory/MemWashAndReturnRight/2025-07-25-00-45-53/demo_im128_notp.hdf5",
+        "memory/MemHeatPotMultiple/2025-09-20-20-10-55/demo_im128_notp.hdf5",
+        "memory/MemHeatPotMultiple/2025-09-20-20-35-24/demo_im128_notp.hdf5",
+        "memory/MemHeatPotMultiple/2025-09-20-20-50-23/demo_im128_notp.hdf5",
+        "memory/MemHeatPotMultiple/2025-09-20-21-08-42/demo_im128_notp.hdf5",
+        "memory/MemHeatPotMultiple/2025-09-20-21-35-46/demo_im128_notp.hdf5",
+        "memory/MemPutKBowlInCabinet/seed0_2025-09-19-23-10-47_6demo/demo_im128_notp.hdf5",
+        "memory/MemPutKBowlInCabinet/seed1_2025-09-19-23-32-29_20demo/demo_im128_notp.hdf5",
+        "memory/MemPutKBowlInCabinet/seed2_2025-09-20-00-42-49_30demo/demo_im128_notp.hdf5",
+        "memory/MemRetrieveOilsFromCounterLL/2025-09-20-22-49-27/demo_im128_notp.hdf5",
+        "memory/MemRetrieveOilsFromCounterLR/2025-09-20-23-25-19/demo_im128_notp.hdf5",
+        "memory/MemRetrieveOilsFromCounterRL/2025-09-20-23-06-15/demo_im128_notp.hdf5",
+        "memory/MemRetrieveOilsFromCounterRR/2025-09-20-23-14-26/demo_im128_notp.hdf5",
+        "memory/MemWashAndReturnSameLocation/2025-09-20-21-52-31/demo_im128_notp.hdf5",
+        "memory/MemWashAndReturnSameLocation/2025-09-20-22-11-00/demo_im128_notp.hdf5",
+        "memory/MemWashAndReturnSameLocation/2025-09-20-22-25-19/demo_im128_notp.hdf5"
+    ],
+    "image_keys": [
+        "obs/robot0_agentview_center_image",
+        "obs/robot0_eye_in_hand_image"
+    ],
+    "max_traj_len": 2048,
+    "low_dim_keys": [],
+    "group_method": "dinov2",
+    "proprio_keys": [
+        "obs/robot0_joint_pos_cos",
+        "obs/robot0_gripper_qpos"
+    ],
+    "action_keys": [
+        "actions"
+    ]
+}