Upload folder using huggingface_hub

.gitattributes

@@ -46,3 +46,6 @@ train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 train_2/wandb/debug-internal.log filter=lfs diff=lfs merge=lfs -text
 train_2/wandb/run-20241120_221447-3pyqkvto_train_2/logs/debug-internal.log filter=lfs diff=lfs merge=lfs -text
 train_2/wandb/run-20241120_221447-3pyqkvto_train_2/run-3pyqkvto_train_2.wandb filter=lfs diff=lfs merge=lfs -text
+Lift/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Lift/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Lift/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text

Lift/config.yaml

@@ -0,0 +1,280 @@
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+checkpoint:
+  save_last_ckpt: false
+  save_last_snapshot: false
+  topk:
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+    k: 1
+    mode: max
+    monitor_key: test_mean_score
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: true
+dataset_obs_steps: 2
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  inv_gamma: 1.0
+  max_value: 0.9999
+  min_value: 0.0
+  power: 0.75
+  update_after_step: 0
+exp_name: default
+horizon: 20
+keypoint_visible_rate: 1.0
+logging:
+  group: null
+  id: null
+  mode: online
+  name: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image
+  project: diffusion_policy_debug
+  resume: true
+  tags:
+  - train_tedi_unet_hybrid
+  - lift_image
+  - default
+multi_run:
+  run_dir: data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image
+  wandb_name_base: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image
+n_action_steps: 8
+n_latency_steps: 0
+n_obs_steps: 2
+name: train_tedi_unet_hybrid
+obs_as_global_cond: true
+optimizer:
+  _target_: torch.optim.AdamW
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  lr: 0.0001
+  weight_decay: 1.0e-06
+past_action_visible: false
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  buffer_init: zero
+  chunk_wise_weight: 0.8
+  cond_predict_scale: true
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  eval_fixed_crop: true
+  horizon: 20
+  kernel_size: 5
+  n_action_steps: 8
+  n_groups: 8
+  n_obs_steps: 2
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    beta_start: 0.0001
+    clip_sample: true
+    num_train_timesteps: 100
+    prediction_type: epsilon
+    variance_type: fixed_small
+  num_inference_steps: 100
+  obs_as_global_cond: true
+  obs_encoder_group_norm: true
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+shape_meta:
+  action:
+    shape:
+    - 10
+  obs:
+    agentview_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_eef_pos:
+      shape:
+      - 3
+    robot0_eef_quat:
+      shape:
+      - 4
+    robot0_eye_in_hand_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_gripper_qpos:
+      shape:
+      - 2
+task:
+  abs_action: true
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    abs_action: true
+    dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
+    horizon: 20
+    n_obs_steps: 2
+    pad_after: 7
+    pad_before: 1
+    rotation_rep: rotation_6d
+    seed: 42
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    use_cache: true
+    use_legacy_normalizer: false
+    val_ratio: 0.02
+  dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
+  dataset_type: ph
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    abs_action: true
+    crf: 22
+    dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
+    fps: 10
+    max_steps: 400
+    n_action_steps: 8
+    n_envs: 28
+    n_obs_steps: 2
+    n_test: 50
+    n_test_vis: 4
+    n_train: 6
+    n_train_vis: 2
+    past_action: false
+    render_obs_key: agentview_image
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    test_start_seed: 100000
+    tqdm_interval_sec: 1.0
+    train_start_idx: 0
+  name: lift_image
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  task_name: lift
+task_name: lift_image
+training:
+  checkpoint_every: 50
+  debug: false
+  device: cuda:0
+  gradient_accumulate_every: 1
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  max_train_steps: null
+  max_val_steps: null
+  num_epochs: 1501
+  resume: true
+  rollout_every: 50
+  sample_every: 5
+  seed: 42
+  tqdm_interval_sec: 1.0
+  use_ema: true
+  val_every: 1
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: false

Lift/metrics/logs.json.txt

@@ -0,0 +1,31 @@
+{"global_step": 114, "epoch": 0, "max/test_mean_score": 0.04, "k_around_max/test_mean_score": 0.04, "max_k_window/test_mean_score": 0.04, "min_train_loss/test_mean_score": 0.04, "min_val_loss/test_mean_score": 0.04, "k_min_train_loss/test_mean_score": 0.04, "k_min_val_loss/test_mean_score": 0.04, "last/test_mean_score": 0.04, "metric_global_step/test_mean_score": 0}
+{"global_step": 5864, "epoch": 50, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.52, "max_k_window/test_mean_score": 0.52, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.52, "k_min_val_loss/test_mean_score": 0.52, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 1}
+{"global_step": 11614, "epoch": 100, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.6777777777777777, "max_k_window/test_mean_score": 0.6126666666666667, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.6777777777777777, "k_min_val_loss/test_mean_score": 0.6777777777777777, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 2}
+{"global_step": 17364, "epoch": 150, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.7583333333333334, "max_k_window/test_mean_score": 0.7113333333333332, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.7583333333333334, "k_min_val_loss/test_mean_score": 0.7583333333333334, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 3}
+{"global_step": 23114, "epoch": 200, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8066666666666666, "max_k_window/test_mean_score": 0.8073333333333332, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.8066666666666666, "k_min_val_loss/test_mean_score": 0.8066666666666666, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 4}
+{"global_step": 28864, "epoch": 250, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.8999999999999998, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.8377777777777777, "k_min_val_loss/test_mean_score": 0.8377777777777777, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 5}
+{"global_step": 34614, "epoch": 300, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9986666666666665, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.8609523809523809, "k_min_val_loss/test_mean_score": 0.8609523809523809, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 6}
+{"global_step": 40364, "epoch": 350, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9986666666666665, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.8783333333333334, "k_min_val_loss/test_mean_score": 0.8783333333333334, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 7}
+{"global_step": 46114, "epoch": 400, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9993333333333333, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.8918518518518518, "k_min_val_loss/test_mean_score": 0.8918518518518518, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 8}
+{"global_step": 51864, "epoch": 450, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9993333333333333, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9026666666666667, "k_min_val_loss/test_mean_score": 0.9026666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 9}
+{"global_step": 57614, "epoch": 500, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9993333333333333, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9986666666666667, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 10}
+{"global_step": 63364, "epoch": 550, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9986666666666667, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 11}
+{"global_step": 69114, "epoch": 600, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9993333333333334, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 12}
+{"global_step": 74864, "epoch": 650, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9993333333333334, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 13}
+{"global_step": 80614, "epoch": 700, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 0.9993333333333334, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 14}
+{"global_step": 86364, "epoch": 750, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 15}
+{"global_step": 92114, "epoch": 800, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 16}
+{"global_step": 97864, "epoch": 850, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 17}
+{"global_step": 103614, "epoch": 900, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 18}
+{"global_step": 109364, "epoch": 950, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 19}
+{"global_step": 115114, "epoch": 1000, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 20}
+{"global_step": 120864, "epoch": 1050, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 21}
+{"global_step": 126614, "epoch": 1100, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 22}
+{"global_step": 132364, "epoch": 1150, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 23}
+{"global_step": 138114, "epoch": 1200, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 24}
+{"global_step": 143864, "epoch": 1250, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 25}
+{"global_step": 149614, "epoch": 1300, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 26}
+{"global_step": 155364, "epoch": 1350, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 27}
+{"global_step": 161114, "epoch": 1400, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 28}
+{"global_step": 166864, "epoch": 1450, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 29}
+{"global_step": 172614, "epoch": 1500, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.8377777777777777, "max_k_window/test_mean_score": 0.9999999999999999, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 1.0, "k_min_train_loss/test_mean_score": 1.0, "k_min_val_loss/test_mean_score": 0.9986666666666667, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 30}

Lift/metrics/metrics.json

@@ -0,0 +1,13 @@
+{
+  "epoch": 1500,
+  "global_step": 172614,
+  "k_around_max/test_mean_score": 0.8377777777777777,
+  "k_min_train_loss/test_mean_score": 1.0,
+  "k_min_val_loss/test_mean_score": 0.9986666666666667,
+  "last/test_mean_score": 1.0,
+  "max/test_mean_score": 1.0,
+  "max_k_window/test_mean_score": 0.9999999999999999,
+  "metric_global_step/test_mean_score": 30,
+  "min_train_loss/test_mean_score": 1.0,
+  "min_val_loss/test_mean_score": 1.0
+}

Lift/metrics/metrics.log

@@ -0,0 +1,39 @@
+2024-11-20 22:14:22,147 [INFO] Monitor waiting for log files!
+2024-11-20 22:14:54,338 [INFO] All log files ready!
+2024-11-20 22:18:58,395 [INFO] Metrics logged at step 114
+2024-11-20 22:33:35,212 [INFO] Metrics logged at step 5864
+2024-11-20 22:48:12,539 [INFO] Metrics logged at step 11614
+2024-11-20 23:02:39,974 [INFO] Metrics logged at step 17364
+2024-11-20 23:16:57,668 [INFO] Metrics logged at step 23114
+2024-11-20 23:31:16,078 [INFO] Metrics logged at step 28864
+2024-11-20 23:45:35,079 [INFO] Metrics logged at step 34614
+2024-11-20 23:59:44,369 [INFO] Metrics logged at step 40364
+2024-11-21 00:13:33,875 [INFO] Metrics logged at step 46114
+2024-11-21 00:27:33,530 [INFO] Metrics logged at step 51864
+2024-11-21 00:41:23,592 [INFO] Metrics logged at step 57614
+2024-11-21 00:55:24,357 [INFO] Metrics logged at step 63364
+2024-11-21 01:09:05,254 [INFO] Metrics logged at step 69114
+2024-11-21 01:22:46,131 [INFO] Metrics logged at step 74864
+2024-11-21 01:36:37,706 [INFO] Metrics logged at step 80614
+2024-11-21 01:50:30,104 [INFO] Metrics logged at step 86364
+2024-11-21 02:04:22,272 [INFO] Metrics logged at step 92114
+2024-11-21 02:18:04,721 [INFO] Metrics logged at step 97864
+2024-11-21 02:31:57,666 [INFO] Metrics logged at step 103614
+2024-11-21 02:45:41,000 [INFO] Metrics logged at step 109364
+2024-11-21 02:59:14,624 [INFO] Metrics logged at step 115114
+2024-11-21 03:13:19,456 [INFO] Metrics logged at step 120864
+2024-11-21 03:27:14,100 [INFO] Metrics logged at step 126614
+2024-11-21 03:41:09,312 [INFO] Metrics logged at step 132364
+2024-11-21 03:54:54,118 [INFO] Metrics logged at step 138114
+2024-11-21 04:08:29,665 [INFO] Metrics logged at step 143864
+2024-11-21 04:22:05,472 [INFO] Metrics logged at step 149614
+2024-11-21 04:35:41,397 [INFO] Metrics logged at step 155364
+2024-11-21 04:49:17,982 [INFO] Metrics logged at step 161114
+2024-11-21 05:02:55,251 [INFO] Metrics logged at step 166864
+2024-11-21 05:16:11,780 [INFO] Metrics logged at step 172614
+2024-11-21 05:16:25,081 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 05:16:25,098 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 05:16:25,114 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 05:16:25,146 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 05:16:25,162 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 05:16:25,179 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Lift/train_0/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 42
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_0
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 3pyqkvto_train_0
+  group: 3pyqkvto
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: lift_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: lift
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4300000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Lift/train_0/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_0
+    - hydra.mode=RUN
+    task:
+    - task=lift_image_abs
+    - training.seed=42
+    - task.env_runner.test_start_seed=4300000
+    - logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_0
+    - logging.id=3pyqkvto_train_0
+    - logging.group=3pyqkvto
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=3pyqkvto,logging.id=3pyqkvto_train_0,logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_0,task.env_runner.test_start_seed=4300000,task=lift_image_abs,training.seed=42
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_0
+    choices:
+      task: lift_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Lift/train_0/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=lift_image_abs
+- training.seed=42
+- task.env_runner.test_start_seed=4300000
+- logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_0
+- logging.id=3pyqkvto_train_0
+- logging.group=3pyqkvto

Lift/train_0/checkpoints/epoch=0050-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:664ac2a09b4fc5e205c3447886eba9545038f9933bfa82cd9b572cbea607c848
+size 4647697621

Lift/train_0/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7757292b6ffbbbd9f2181b57dc2c413b573ef129e57b0e49e1c3083481ce879
+size 17957755

Lift/train_0/train.log

@@ -0,0 +1,231 @@
+[2024-11-20 22:14:30,109][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-20 22:14:41,110][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,111][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,123][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,123][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,133][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,133][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,143][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,143][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,147][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,148][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,154][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,155][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,164][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,164][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,173][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,173][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,183][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,184][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,197][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,198][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,216][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,217][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,218][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,219][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,222][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,223][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,227][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,229][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,231][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,232][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,242][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,243][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,265][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,266][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,271][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,272][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,276][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,277][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,279][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,280][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,285][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,286][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,290][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,291][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,304][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,305][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,308][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,309][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,326][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,327][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,410][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,411][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,431][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,431][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,445][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,445][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,450][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,450][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,458][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,458][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,463][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,464][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,480][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,481][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,492][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,492][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,495][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,495][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,499][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,500][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,511][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,511][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,520][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,520][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,540][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,540][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,542][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,542][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,550][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,550][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,562][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,563][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,571][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,571][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,578][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,578][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,593][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,593][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,602][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,602][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,644][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,644][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,657][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,657][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,659][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,659][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,659][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,659][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,687][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,687][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,695][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,696][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,700][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,700][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,714][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,714][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,719][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,719][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,730][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,730][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,730][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,731][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,766][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,766][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,799][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,799][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,803][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,803][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,807][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,808][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,813][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,813][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,825][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,825][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,828][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,828][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,834][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,835][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,844][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,844][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,861][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,862][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,864][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,864][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,884][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,884][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,901][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,901][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,908][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,909][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,927][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,927][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,937][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,937][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,940][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,941][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,967][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,967][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,992][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,993][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,995][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,996][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,012][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,013][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,022][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,023][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,036][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,037][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,045][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,045][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,047][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,048][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,052][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,052][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,060][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,060][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,064][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,065][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,079][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,080][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,092][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,093][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,098][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,099][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,110][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,110][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,117][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,117][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,137][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,137][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,154][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,155][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,158][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,159][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,178][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,178][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,184][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,184][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,205][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,205][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,220][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,220][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,223][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,223][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,242][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,243][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,255][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,256][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,257][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,257][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,277][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,277][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,283][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,284][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,290][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,290][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,290][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,291][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,293][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,293][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,308][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,308][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,318][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,318][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,320][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,320][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,353][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,353][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,359][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,359][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,365][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,365][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,460][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,461][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:16:16,821][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:16:16,849][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:16:16,865][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:16:16,899][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:16:16,916][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:16:16,932][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Lift/train_1/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 43
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 3pyqkvto_train_1
+  group: 3pyqkvto
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: lift_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: lift
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4400000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Lift/train_1/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1
+    - hydra.mode=RUN
+    task:
+    - task=lift_image_abs
+    - training.seed=43
+    - task.env_runner.test_start_seed=4400000
+    - logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1
+    - logging.id=3pyqkvto_train_1
+    - logging.group=3pyqkvto
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=3pyqkvto,logging.id=3pyqkvto_train_1,logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1,task.env_runner.test_start_seed=4400000,task=lift_image_abs,training.seed=43
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1
+    choices:
+      task: lift_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Lift/train_1/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=lift_image_abs
+- training.seed=43
+- task.env_runner.test_start_seed=4400000
+- logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1
+- logging.id=3pyqkvto_train_1
+- logging.group=3pyqkvto

Lift/train_1/checkpoints/epoch=0050-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b50a37215becb9f1454df5df57bd4e8fd8fc79074d648a27a073515535f68e3
+size 4647697621

Lift/train_1/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead576c8c923ffecfeb1f984f9c1c6827d653ac67aa65b40e210840da1402a36
+size 17958083

Lift/train_1/train.log

@@ -0,0 +1,231 @@
+[2024-11-20 22:14:30,103][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-20 22:14:41,104][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,104][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,105][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,105][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,105][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,106][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,108][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,108][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,117][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,118][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,130][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,131][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,136][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,137][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,137][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,137][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,151][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,151][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,168][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,169][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,173][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,173][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,182][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,183][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,190][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,193][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,193][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,205][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,206][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,211][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,212][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,236][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,236][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,243][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,243][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,251][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,252][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,256][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,256][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,281][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,282][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,283][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,285][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,303][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,304][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,328][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,328][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,394][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,395][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,399][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,400][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,416][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,416][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,420][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,420][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,422][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,423][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,436][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,437][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,444][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,445][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,452][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,452][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,453][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,454][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,459][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,459][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,460][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,461][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,466][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,467][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,473][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,473][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,484][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,484][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,485][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,485][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,505][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,505][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,530][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,530][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,553][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,553][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,560][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,560][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,571][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,572][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,580][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,581][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,593][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,593][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,601][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,602][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,625][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,668][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,668][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,668][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,668][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,676][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,676][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,683][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,683][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,692][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,692][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,695][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,695][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,704][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,704][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,708][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,709][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,720][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,721][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,725][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,725][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,735][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,736][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,742][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,742][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,749][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,750][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,761][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,761][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,776][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,777][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,784][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,785][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,786][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,786][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,789][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,789][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,831][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,831][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,832][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,832][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,870][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,870][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,873][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,874][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,878][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,879][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,890][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,890][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,891][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,892][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,897][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,897][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,935][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,936][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,962][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,962][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,979][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,979][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,986][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,987][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,994][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,995][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,028][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,028][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,029][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,029][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,038][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,038][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,042][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,043][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,047][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,048][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,068][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,068][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,070][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,071][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,077][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,077][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,082][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,083][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,089][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,089][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,124][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,124][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,141][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,142][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,145][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,145][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,151][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,151][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,162][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,162][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,172][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,173][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,178][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,178][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,221][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,221][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,236][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,236][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,242][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,242][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,250][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,251][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,254][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,255][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,264][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,264][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,270][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,270][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,300][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,300][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,301][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,301][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,310][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,310][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,324][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,324][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,332][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,333][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,333][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,333][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,362][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,362][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,417][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,417][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,495][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,495][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:05:07,557][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:05:07,586][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:05:07,603][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:05:07,636][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:05:07,653][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:05:07,669][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Lift/train_2/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 44
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_2
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 3pyqkvto_train_2
+  group: 3pyqkvto
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: lift_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: lift
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4500000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Lift/train_2/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_2
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_2
+    - hydra.mode=RUN
+    task:
+    - task=lift_image_abs
+    - training.seed=44
+    - task.env_runner.test_start_seed=4500000
+    - logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_2
+    - logging.id=3pyqkvto_train_2
+    - logging.group=3pyqkvto
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=3pyqkvto,logging.id=3pyqkvto_train_2,logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_2,task.env_runner.test_start_seed=4500000,task=lift_image_abs,training.seed=44
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_2
+    choices:
+      task: lift_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Lift/train_2/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=lift_image_abs
+- training.seed=44
+- task.env_runner.test_start_seed=4500000
+- logging.name=2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_2
+- logging.id=3pyqkvto_train_2
+- logging.group=3pyqkvto

Lift/train_2/checkpoints/epoch=0050-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0c12435b5b624c14a409b64e40b9109981173cb316dad43b35b0cbc3beb8595
+size 4647697621

Lift/train_2/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576000f7b3382e5ef65acee2edb16703f043c6d85302c1b783d8266d3ea185aa
+size 17958381

Lift/train_2/train.log

@@ -0,0 +1,231 @@
+[2024-11-20 22:14:30,112][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-20 22:14:41,104][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,105][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,105][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,105][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,113][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,114][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,120][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,120][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,127][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,127][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,127][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,128][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,139][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,140][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,145][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,145][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,156][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,157][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,159][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,160][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,164][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,164][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,175][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,176][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,179][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,179][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,186][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,188][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,200][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,201][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,206][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,206][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,209][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,210][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,217][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,218][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,224][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,225][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,237][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,238][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,248][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,249][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,249][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,250][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,257][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,258][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,260][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,261][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,265][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,265][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,299][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,300][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,335][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,336][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,409][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,410][root][INFO] - Device 0 is not available for rendering
+[2024-11-20 22:14:41,425][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,425][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,428][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,428][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,433][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,434][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,463][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,463][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,467][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,467][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,470][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,470][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,477][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,477][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,478][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,478][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,491][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,492][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,501][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,502][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,508][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,508][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,516][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,517][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,527][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,528][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,533][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,533][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,535][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,535][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,546][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,547][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,556][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,556][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,584][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,585][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,589][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,590][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,598][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,599][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,605][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,605][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,628][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,628][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,633][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,633][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,635][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,636][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,645][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,645][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,664][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,664][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,676][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,676][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,688][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,688][root][INFO] - Device 1 is not available for rendering
+[2024-11-20 22:14:41,738][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,739][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,746][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,747][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,757][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,757][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,767][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,767][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,769][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,770][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,775][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,775][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,800][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,800][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,800][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,800][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,811][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,811][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,817][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,817][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,819][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,819][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,840][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,840][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,857][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,858][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,861][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,862][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,870][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,870][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,878][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,878][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,886][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,886][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,905][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,905][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,912][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,912][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,920][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,921][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,933][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,933][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,938][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,938][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,945][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,946][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,951][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,952][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,962][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,963][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,992][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,992][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:41,993][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:41,993][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,019][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,019][root][INFO] - Device 2 is not available for rendering
+[2024-11-20 22:14:42,066][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,067][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,074][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,075][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,079][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,079][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,101][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,101][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,106][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,106][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,111][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,111][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,126][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,127][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,136][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,137][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,147][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,147][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,163][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,163][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,166][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,166][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,171][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,172][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,188][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,188][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,195][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,195][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,199][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,199][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,202][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,202][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,205][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,205][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,212][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,212][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,213][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,213][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,227][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,227][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,228][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,228][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,235][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,235][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,260][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,260][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,267][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,267][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,274][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,274][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,281][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,281][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,304][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,305][root][INFO] - Device 3 is not available for rendering
+[2024-11-20 22:14:42,313][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-20 22:14:42,313][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 05:12:39,525][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:12:39,559][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:12:39,576][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:12:39,609][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:12:39,626][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 05:12:39,642][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/