Upload folder using huggingface_hub

.gitattributes

@@ -58,3 +58,6 @@ Square/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 PushT/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 PushT/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 PushT/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Transport/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Transport/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Transport/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text

Transport/config.yaml

@@ -0,0 +1,385 @@
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+checkpoint:
+  save_last_ckpt: false
+  save_last_snapshot: false
+  topk:
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+    k: 1
+    mode: max
+    monitor_key: test_mean_score
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: true
+dataset_obs_steps: 2
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  inv_gamma: 1.0
+  max_value: 0.9999
+  min_value: 0.0
+  power: 0.75
+  update_after_step: 0
+exp_name: default
+horizon: 20
+keypoint_visible_rate: 1.0
+logging:
+  group: null
+  id: null
+  mode: online
+  name: 2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image
+  project: diffusion_policy_debug
+  resume: true
+  tags:
+  - train_tedi_unet_hybrid
+  - transport_image
+  - default
+multi_run:
+  run_dir: data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image
+  wandb_name_base: 2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image
+n_action_steps: 8
+n_latency_steps: 0
+n_obs_steps: 2
+name: train_tedi_unet_hybrid
+obs_as_global_cond: true
+optimizer:
+  _target_: torch.optim.AdamW
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  lr: 0.0001
+  weight_decay: 1.0e-06
+past_action_visible: false
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  buffer_init: zero
+  chunk_wise_weight: 0.8
+  cond_predict_scale: true
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  eval_fixed_crop: true
+  horizon: 20
+  kernel_size: 5
+  n_action_steps: 8
+  n_groups: 8
+  n_obs_steps: 2
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    beta_start: 0.0001
+    clip_sample: true
+    num_train_timesteps: 100
+    prediction_type: epsilon
+    variance_type: fixed_small
+  num_inference_steps: 100
+  obs_as_global_cond: true
+  obs_encoder_group_norm: true
+  shape_meta:
+    action:
+      shape:
+      - 20
+    obs:
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+      robot1_eef_pos:
+        shape:
+        - 3
+      robot1_eef_quat:
+        shape:
+        - 4
+      robot1_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_gripper_qpos:
+        shape:
+        - 2
+      shouldercamera0_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      shouldercamera1_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+shape_meta:
+  action:
+    shape:
+    - 20
+  obs:
+    robot0_eef_pos:
+      shape:
+      - 3
+    robot0_eef_quat:
+      shape:
+      - 4
+    robot0_eye_in_hand_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_gripper_qpos:
+      shape:
+      - 2
+    robot1_eef_pos:
+      shape:
+      - 3
+    robot1_eef_quat:
+      shape:
+      - 4
+    robot1_eye_in_hand_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot1_gripper_qpos:
+      shape:
+      - 2
+    shouldercamera0_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    shouldercamera1_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+task:
+  abs_action: true
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    abs_action: true
+    dataset_path: data/robomimic/datasets/transport/ph/image_abs.hdf5
+    horizon: 20
+    n_obs_steps: 2
+    pad_after: 7
+    pad_before: 1
+    rotation_rep: rotation_6d
+    seed: 42
+    shape_meta:
+      action:
+        shape:
+        - 20
+      obs:
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+    use_cache: true
+    use_legacy_normalizer: false
+    val_ratio: 0.02
+  dataset_path: data/robomimic/datasets/transport/ph/image_abs.hdf5
+  dataset_type: ph
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    abs_action: true
+    crf: 22
+    dataset_path: data/robomimic/datasets/transport/ph/image_abs.hdf5
+    fps: 10
+    max_steps: 700
+    n_action_steps: 8
+    n_envs: 28
+    n_obs_steps: 2
+    n_test: 50
+    n_test_vis: 4
+    n_train: 6
+    n_train_vis: 2
+    past_action: false
+    render_obs_key: shouldercamera0_image
+    shape_meta:
+      action:
+        shape:
+        - 20
+      obs:
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+    test_start_seed: 100000
+    tqdm_interval_sec: 1.0
+    train_start_idx: 0
+  name: transport_image
+  shape_meta:
+    action:
+      shape:
+      - 20
+    obs:
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+      robot1_eef_pos:
+        shape:
+        - 3
+      robot1_eef_quat:
+        shape:
+        - 4
+      robot1_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_gripper_qpos:
+        shape:
+        - 2
+      shouldercamera0_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      shouldercamera1_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+  task_name: transport
+task_name: transport_image
+training:
+  checkpoint_every: 50
+  debug: false
+  device: cuda:0
+  gradient_accumulate_every: 1
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  max_train_steps: null
+  max_val_steps: null
+  num_epochs: 501
+  resume: true
+  rollout_every: 50
+  sample_every: 5
+  seed: 42
+  tqdm_interval_sec: 1.0
+  use_ema: true
+  val_every: 1
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: false

Transport/metrics/logs.json.txt

@@ -0,0 +1,11 @@
+{"global_step": 1402, "epoch": 0, "max/test_mean_score": 0.0, "k_around_max/test_mean_score": 0.0, "max_k_window/test_mean_score": 0.0, "min_train_loss/test_mean_score": 0.0, "min_val_loss/test_mean_score": 0.0, "k_min_train_loss/test_mean_score": 0.0, "k_min_val_loss/test_mean_score": 0.0, "last/test_mean_score": 0.0, "metric_global_step/test_mean_score": 0}
+{"global_step": 71552, "epoch": 50, "max/test_mean_score": 0.8000000000000002, "k_around_max/test_mean_score": 0.4000000000000001, "max_k_window/test_mean_score": 0.4000000000000001, "min_train_loss/test_mean_score": 0.8000000000000002, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.4000000000000001, "k_min_val_loss/test_mean_score": 0.4000000000000001, "last/test_mean_score": 0.8000000000000002, "metric_global_step/test_mean_score": 1}
+{"global_step": 141702, "epoch": 100, "max/test_mean_score": 0.8533333333333334, "k_around_max/test_mean_score": 0.5511111111111111, "max_k_window/test_mean_score": 0.5066666666666667, "min_train_loss/test_mean_score": 0.8533333333333334, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.5511111111111111, "k_min_val_loss/test_mean_score": 0.5511111111111111, "last/test_mean_score": 0.8533333333333334, "metric_global_step/test_mean_score": 2}
+{"global_step": 211852, "epoch": 150, "max/test_mean_score": 0.88, "k_around_max/test_mean_score": 0.6333333333333334, "max_k_window/test_mean_score": 0.6053333333333334, "min_train_loss/test_mean_score": 0.88, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.6333333333333334, "k_min_val_loss/test_mean_score": 0.6333333333333334, "last/test_mean_score": 0.88, "metric_global_step/test_mean_score": 3}
+{"global_step": 282002, "epoch": 200, "max/test_mean_score": 0.88, "k_around_max/test_mean_score": 0.6813333333333332, "max_k_window/test_mean_score": 0.6900000000000001, "min_train_loss/test_mean_score": 0.8733333333333334, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.6813333333333333, "k_min_val_loss/test_mean_score": 0.6813333333333332, "last/test_mean_score": 0.8733333333333334, "metric_global_step/test_mean_score": 4}
+{"global_step": 352152, "epoch": 250, "max/test_mean_score": 0.88, "k_around_max/test_mean_score": 0.7022222222222222, "max_k_window/test_mean_score": 0.7439999999999999, "min_train_loss/test_mean_score": 0.8066666666666666, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.7022222222222222, "k_min_val_loss/test_mean_score": 0.7022222222222222, "last/test_mean_score": 0.8066666666666666, "metric_global_step/test_mean_score": 5}
+{"global_step": 422302, "epoch": 300, "max/test_mean_score": 0.88, "k_around_max/test_mean_score": 0.7209523809523809, "max_k_window/test_mean_score": 0.8380000000000001, "min_train_loss/test_mean_score": 0.8333333333333334, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.7209523809523809, "k_min_val_loss/test_mean_score": 0.7209523809523809, "last/test_mean_score": 0.8333333333333334, "metric_global_step/test_mean_score": 6}
+{"global_step": 492452, "epoch": 350, "max/test_mean_score": 0.8866666666666667, "k_around_max/test_mean_score": 0.7400000000000001, "max_k_window/test_mean_score": 0.8613333333333334, "min_train_loss/test_mean_score": 0.8733333333333334, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.7400000000000001, "k_min_val_loss/test_mean_score": 0.7400000000000001, "last/test_mean_score": 0.8733333333333334, "metric_global_step/test_mean_score": 7}
+{"global_step": 562602, "epoch": 400, "max/test_mean_score": 0.8866666666666667, "k_around_max/test_mean_score": 0.7449074074074075, "max_k_window/test_mean_score": 0.852, "min_train_loss/test_mean_score": 0.8466666666666667, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.7518518518518519, "k_min_val_loss/test_mean_score": 0.7518518518518519, "last/test_mean_score": 0.8466666666666667, "metric_global_step/test_mean_score": 8}
+{"global_step": 632752, "epoch": 450, "max/test_mean_score": 0.8866666666666667, "k_around_max/test_mean_score": 0.7488333333333334, "max_k_window/test_mean_score": 0.852, "min_train_loss/test_mean_score": 0.8466666666666667, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.7613333333333334, "k_min_val_loss/test_mean_score": 0.7613333333333334, "last/test_mean_score": 0.8466666666666667, "metric_global_step/test_mean_score": 9}
+{"global_step": 702902, "epoch": 500, "max/test_mean_score": 0.8866666666666667, "k_around_max/test_mean_score": 0.7781666666666668, "max_k_window/test_mean_score": 0.858, "min_train_loss/test_mean_score": 0.8666666666666667, "min_val_loss/test_mean_score": 0.8000000000000002, "k_min_train_loss/test_mean_score": 0.848, "k_min_val_loss/test_mean_score": 0.848, "last/test_mean_score": 0.8666666666666667, "metric_global_step/test_mean_score": 10}

Transport/metrics/metrics.json

@@ -0,0 +1,13 @@
+{
+  "epoch": 500,
+  "global_step": 702902,
+  "k_around_max/test_mean_score": 0.7781666666666668,
+  "k_min_train_loss/test_mean_score": 0.848,
+  "k_min_val_loss/test_mean_score": 0.848,
+  "last/test_mean_score": 0.8666666666666667,
+  "max/test_mean_score": 0.8866666666666667,
+  "max_k_window/test_mean_score": 0.858,
+  "metric_global_step/test_mean_score": 10,
+  "min_train_loss/test_mean_score": 0.8666666666666667,
+  "min_val_loss/test_mean_score": 0.8000000000000002
+}

Transport/metrics/metrics.log

@@ -0,0 +1,19 @@
+2024-11-21 05:17:14,200 [INFO] Monitor waiting for log files!
+2024-11-21 05:18:25,571 [INFO] All log files ready!
+2024-11-21 05:44:06,353 [INFO] Metrics logged at step 1402
+2024-11-21 08:40:28,284 [INFO] Metrics logged at step 71552
+2024-11-21 11:37:27,703 [INFO] Metrics logged at step 141702
+2024-11-21 14:34:41,659 [INFO] Metrics logged at step 211852
+2024-11-21 17:31:44,258 [INFO] Metrics logged at step 282002
+2024-11-21 20:27:16,684 [INFO] Metrics logged at step 352152
+2024-11-21 23:22:40,980 [INFO] Metrics logged at step 422302
+2024-11-22 02:18:45,469 [INFO] Metrics logged at step 492452
+2024-11-22 05:14:54,501 [INFO] Metrics logged at step 562602
+2024-11-22 08:10:14,658 [INFO] Metrics logged at step 632752
+2024-11-22 11:00:08,394 [INFO] Metrics logged at step 702902
+2024-11-22 11:00:19,767 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-22 11:00:19,785 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-22 11:00:19,802 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-22 11:00:19,836 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-22 11:00:19,852 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-22 11:00:19,869 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Transport/train_0/.hydra/config.yaml

@@ -0,0 +1,293 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 42
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_0
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 8th7h1v9_train_0
+  group: 8th7h1v9
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: transport_image
+  shape_meta:
+    obs:
+      shouldercamera0_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+      shouldercamera1_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eef_pos:
+        shape:
+        - 3
+      robot1_eef_quat:
+        shape:
+        - 4
+      robot1_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 20
+  task_name: transport
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4300000
+    max_steps: 700
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: shouldercamera0_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Transport/train_0/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_0
+    - hydra.mode=RUN
+    task:
+    - task=transport_image_abs
+    - training.seed=42
+    - task.env_runner.test_start_seed=4300000
+    - logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_0
+    - logging.id=8th7h1v9_train_0
+    - logging.group=8th7h1v9
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=8th7h1v9,logging.id=8th7h1v9_train_0,logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_0,task.env_runner.test_start_seed=4300000,task=transport_image_abs,training.seed=42
+    id: ???
+    num: ???
+    config_name: tedi_transport.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_0
+    choices:
+      task: transport_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Transport/train_0/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=transport_image_abs
+- training.seed=42
+- task.env_runner.test_start_seed=4300000
+- logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_0
+- logging.id=8th7h1v9_train_0
+- logging.group=8th7h1v9

Transport/train_0/checkpoints/epoch=0350-test_mean_score=0.880.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b95134ac2a66ac3402224e4a4b86674ec9784642f71d8c818b77ef50c94eb3e9
+size 5311705609

Transport/train_0/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:951e31785081290263ea91933f5dbc2a4a7eee46e617689559635a84bc985595
+size 72764307

Transport/train_0/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 05:17:22,216][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.647492e+08
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,130][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,130][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,130][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,131][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,130][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,132][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,215][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,216][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,267][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,267][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,352][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,352][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,420][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,420][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,422][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,422][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,436][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,436][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,437][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,438][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,441][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,441][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,447][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,447][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,505][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,505][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,524][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,524][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,530][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,531][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,557][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,558][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,629][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,632][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,689][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,689][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,707][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,708][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,726][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,728][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,734][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,735][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,742][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,742][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,745][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,745][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,798][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,799][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,833][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,833][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,835][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,835][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,841][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,841][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:49,849][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,849][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,871][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,872][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:49,885][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,886][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:49,981][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,981][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:49,986][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:49,987][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,029][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,030][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,061][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,061][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,067][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,068][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,079][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,079][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,086][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,086][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,086][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,087][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,146][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,147][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,158][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,158][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,185][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,186][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,192][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,192][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,215][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,215][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,215][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,216][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,256][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,256][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,259][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,259][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,262][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,263][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,319][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,319][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,413][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,414][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,487][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,488][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,533][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,533][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,568][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,568][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,573][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,573][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,572][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,573][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,578][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,578][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,599][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,599][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,627][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,628][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,629][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,629][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,661][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,662][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,712][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,712][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,741][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,741][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,842][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,843][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,872][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,872][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,881][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,881][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:50,937][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,938][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,941][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,941][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:50,956][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,957][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:50,960][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,960][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:50,995][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:50,995][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,057][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,057][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,060][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,060][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,112][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,112][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,133][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,134][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,134][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,135][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,190][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,213][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,213][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,254][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,254][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,319][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,320][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,359][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,359][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,451][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,452][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,462][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,462][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,474][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,475][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,496][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,496][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,498][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,498][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,555][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,555][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,562][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,563][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,563][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,563][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,587][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,587][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,610][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,611][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,708][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,708][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,717][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,718][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:17:51,782][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,783][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,850][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,851][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,853][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,853][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:51,891][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,892][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:51,920][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,920][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,925][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,925][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,932][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,932][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:51,968][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:51,969][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,017][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,018][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:17:52,094][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,094][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:52,201][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,202][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:52,298][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,298][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,300][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,301][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,303][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,303][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,309][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,309][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:17:52,380][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,380][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,460][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,460][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:17:52,576][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:17:52,576][root][INFO] - Device 3 is not available for rendering
+[2024-11-22 10:45:23,246][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 10:45:23,275][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 10:45:23,292][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 10:45:23,325][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 10:45:23,342][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 10:45:23,358][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Transport/train_1/.hydra/config.yaml

@@ -0,0 +1,293 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 43
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_1
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 8th7h1v9_train_1
+  group: 8th7h1v9
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: transport_image
+  shape_meta:
+    obs:
+      shouldercamera0_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+      shouldercamera1_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eef_pos:
+        shape:
+        - 3
+      robot1_eef_quat:
+        shape:
+        - 4
+      robot1_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 20
+  task_name: transport
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4400000
+    max_steps: 700
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: shouldercamera0_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Transport/train_1/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_1
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_1
+    - hydra.mode=RUN
+    task:
+    - task=transport_image_abs
+    - training.seed=43
+    - task.env_runner.test_start_seed=4400000
+    - logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_1
+    - logging.id=8th7h1v9_train_1
+    - logging.group=8th7h1v9
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=8th7h1v9,logging.id=8th7h1v9_train_1,logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_1,task.env_runner.test_start_seed=4400000,task=transport_image_abs,training.seed=43
+    id: ???
+    num: ???
+    config_name: tedi_transport.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_1
+    choices:
+      task: transport_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Transport/train_1/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=transport_image_abs
+- training.seed=43
+- task.env_runner.test_start_seed=4400000
+- logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_1
+- logging.id=8th7h1v9_train_1
+- logging.group=8th7h1v9

Transport/train_1/checkpoints/epoch=0150-test_mean_score=0.880.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea57628cc6ebecd79d327c8f51affbc2b337dd4eb92c930c9859c6736236f21b
+size 5311705545

Transport/train_1/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8dffe1371ef51d0a14d0b673abeafcb4fc62e889fe73670e38f336723dfd40c9
+size 72766120

Transport/train_1/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 05:17:22,220][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.647492e+08
+[2024-11-21 05:18:17,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,189][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,235][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,235][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,241][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,242][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,321][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,322][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,356][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,356][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,380][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,380][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,443][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,443][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,488][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,489][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,490][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,491][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,493][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,493][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,566][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,566][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,568][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,568][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,617][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,618][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,636][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,637][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,639][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,640][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,641][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,642][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,679][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,680][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,687][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,687][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,749][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,749][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,783][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,783][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,787][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,787][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,790][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,791][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,794][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,794][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,805][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,805][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,877][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,878][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,878][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,878][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,906][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,906][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,932][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,932][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:17,945][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,945][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,951][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,952][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,953][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,953][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:17,954][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,954][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:17,986][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:17,986][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,030][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,031][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,031][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,031][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,038][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,038][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,045][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,045][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,097][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,097][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,106][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,106][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,129][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,129][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,163][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,164][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,165][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,166][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,170][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,170][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,227][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,227][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,227][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,227][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,227][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,227][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,230][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,231][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,264][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,265][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,284][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,285][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,287][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,287][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,305][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,305][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,323][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,324][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,326][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,326][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,337][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,337][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,339][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,339][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,342][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,343][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,350][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,350][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,350][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,351][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,448][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,448][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,451][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,452][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,456][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,456][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,456][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,456][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,496][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,496][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,524][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,524][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,524][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,524][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,524][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,524][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,560][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,560][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,560][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,561][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,576][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,576][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,636][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,637][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,637][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,637][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,638][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,638][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,669][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,670][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,687][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,687][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,700][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,700][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,701][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,701][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:18,704][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,704][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,718][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,718][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,766][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,766][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,800][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,801][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,803][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,803][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,806][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,806][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,807][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,807][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,808][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,809][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,845][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,845][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,904][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,905][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,909][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,909][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:18,909][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,910][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,909][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,910][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:18,910][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,911][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:18,968][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:18,969][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,023][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,023][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:19,024][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,024][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,039][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,040][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,043][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,044][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:19,086][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,086][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,101][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,102][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,119][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,120][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,138][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,139][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,178][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,178][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,221][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,221][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,280][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,280][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,281][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,281][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,304][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,305][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,323][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,323][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:19,325][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,325][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,412][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,412][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,435][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,436][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,470][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,470][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,544][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,544][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:19,610][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:19,610][root][INFO] - Device 3 is not available for rendering
+[2024-11-22 09:18:03,778][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 09:18:03,854][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 09:18:03,871][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 09:18:03,904][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 09:18:03,921][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 09:18:03,937][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Transport/train_2/.hydra/config.yaml

@@ -0,0 +1,293 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 44
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_2
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 8th7h1v9_train_2
+  group: 8th7h1v9
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: transport_image
+  shape_meta:
+    obs:
+      shouldercamera0_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+      shouldercamera1_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot1_eef_pos:
+        shape:
+        - 3
+      robot1_eef_quat:
+        shape:
+        - 4
+      robot1_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 20
+  task_name: transport
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4500000
+    max_steps: 700
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: shouldercamera0_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        shouldercamera0_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+        shouldercamera1_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot1_eef_pos:
+          shape:
+          - 3
+        robot1_eef_quat:
+          shape:
+          - 4
+        robot1_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 20
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Transport/train_2/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_2
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_2
+    - hydra.mode=RUN
+    task:
+    - task=transport_image_abs
+    - training.seed=44
+    - task.env_runner.test_start_seed=4500000
+    - logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_2
+    - logging.id=8th7h1v9_train_2
+    - logging.group=8th7h1v9
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=8th7h1v9,logging.id=8th7h1v9_train_2,logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_2,task.env_runner.test_start_seed=4500000,task=transport_image_abs,training.seed=44
+    id: ???
+    num: ???
+    config_name: tedi_transport.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_05-16-39_005202_3468147/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/05.16.53_train_tedi_unet_hybrid_transport_image/train_2
+    choices:
+      task: transport_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Transport/train_2/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=transport_image_abs
+- training.seed=44
+- task.env_runner.test_start_seed=4500000
+- logging.name=2024.11.21-05.16.53_train_tedi_unet_hybrid_transport_image_train_2
+- logging.id=8th7h1v9_train_2
+- logging.group=8th7h1v9

Transport/train_2/checkpoints/epoch=0150-test_mean_score=0.900.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a67b85891c99714838eb2e440b6c45ffb83e255fa8f613cad5a311fb912088
+size 5311705545

Transport/train_2/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:925627f916fc78e7ac038d46f61546ce631986ec80a7b6e7ea321183f2baf142
+size 72771281

Transport/train_2/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 05:17:22,221][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.647492e+08
+[2024-11-21 05:18:00,047][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,048][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,049][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,050][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,104][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,104][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,185][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,186][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,263][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,263][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,265][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,265][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,267][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,267][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,369][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,370][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,398][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,398][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,454][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,454][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,454][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,454][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,495][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,495][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,563][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,563][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,601][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,602][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,605][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,605][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,608][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,608][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,611][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,611][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,650][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,651][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,707][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,707][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,709][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,709][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,711][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,711][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,779][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,779][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,783][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,783][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,814][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,814][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,900][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,901][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:00,937][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,938][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,940][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,940][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,954][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,954][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:00,955][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,956][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,972][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,973][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:00,975][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,975][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:00,977][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,978][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:00,980][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,981][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:00,987][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,987][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:00,989][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:00,989][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,106][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,107][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,109][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,109][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,112][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,112][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,190][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,193][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,194][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,255][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,255][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,257][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,258][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,260][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,261][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,293][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,293][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,296][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,296][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,335][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,336][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,345][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,346][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,348][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,349][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,392][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,392][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,427][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,430][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,431][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,444][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,444][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,452][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,453][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,516][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,516][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,549][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,550][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,560][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,560][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,561][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,562][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,585][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,586][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,618][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,618][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,661][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,662][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,671][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,672][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,674][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,675][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,705][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,706][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,715][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,716][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,760][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,761][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:01,803][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,804][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,807][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,807][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,812][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,812][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,816][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,817][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,921][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,921][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,922][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,922][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:01,925][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,925][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:01,955][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,956][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:01,955][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:01,956][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:02,001][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,001][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:02,024][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,024][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,024][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,025][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,059][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,059][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,063][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,063][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:02,064][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,065][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,080][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,080][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,124][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,125][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,136][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,137][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,220][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,220][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,223][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,224][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:02,245][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,246][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,257][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,258][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,260][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,261][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 05:18:02,350][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,350][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,353][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,353][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,394][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,394][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,415][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,415][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,417][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,417][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,427][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,428][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,428][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,428][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,526][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,526][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,610][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,611][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,611][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,611][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 05:18:02,684][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,685][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,687][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,688][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,690][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,690][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,708][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,709][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,784][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,785][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,786][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,787][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,787][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,788][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,889][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,890][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 05:18:02,961][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,962][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,964][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,965][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:02,966][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:02,966][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:03,122][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:03,122][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:18:03,196][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 05:18:03,196][root][INFO] - Device 3 is not available for rendering
+[2024-11-22 11:00:10,173][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 11:00:10,200][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 11:00:10,216][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 11:00:10,249][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 11:00:10,266][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-22 11:00:10,283][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/