Upload folder using huggingface_hub

.gitattributes

@@ -49,3 +49,6 @@ train_2/wandb/run-20241120_221447-3pyqkvto_train_2/run-3pyqkvto_train_2.wandb fi
 Lift/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 Lift/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 Lift/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Can/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Can/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Can/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text

Can/config.yaml

@@ -0,0 +1,280 @@
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+checkpoint:
+  save_last_ckpt: false
+  save_last_snapshot: false
+  topk:
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+    k: 1
+    mode: max
+    monitor_key: test_mean_score
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: true
+dataset_obs_steps: 2
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  inv_gamma: 1.0
+  max_value: 0.9999
+  min_value: 0.0
+  power: 0.75
+  update_after_step: 0
+exp_name: default
+horizon: 20
+keypoint_visible_rate: 1.0
+logging:
+  group: null
+  id: null
+  mode: online
+  name: 2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image
+  project: diffusion_policy_debug
+  resume: true
+  tags:
+  - train_tedi_unet_hybrid
+  - can_image
+  - default
+multi_run:
+  run_dir: data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image
+  wandb_name_base: 2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image
+n_action_steps: 8
+n_latency_steps: 0
+n_obs_steps: 2
+name: train_tedi_unet_hybrid
+obs_as_global_cond: true
+optimizer:
+  _target_: torch.optim.AdamW
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  lr: 0.0001
+  weight_decay: 1.0e-06
+past_action_visible: false
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  buffer_init: zero
+  chunk_wise_weight: 0.8
+  cond_predict_scale: true
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  eval_fixed_crop: true
+  horizon: 20
+  kernel_size: 5
+  n_action_steps: 8
+  n_groups: 8
+  n_obs_steps: 2
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    beta_start: 0.0001
+    clip_sample: true
+    num_train_timesteps: 100
+    prediction_type: epsilon
+    variance_type: fixed_small
+  num_inference_steps: 100
+  obs_as_global_cond: true
+  obs_encoder_group_norm: true
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+shape_meta:
+  action:
+    shape:
+    - 10
+  obs:
+    agentview_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_eef_pos:
+      shape:
+      - 3
+    robot0_eef_quat:
+      shape:
+      - 4
+    robot0_eye_in_hand_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_gripper_qpos:
+      shape:
+      - 2
+task:
+  abs_action: true
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    abs_action: true
+    dataset_path: data/robomimic/datasets/can/ph/image_abs.hdf5
+    horizon: 20
+    n_obs_steps: 2
+    pad_after: 7
+    pad_before: 1
+    rotation_rep: rotation_6d
+    seed: 42
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    use_cache: true
+    use_legacy_normalizer: false
+    val_ratio: 0.02
+  dataset_path: data/robomimic/datasets/can/ph/image_abs.hdf5
+  dataset_type: ph
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    abs_action: true
+    crf: 22
+    dataset_path: data/robomimic/datasets/can/ph/image_abs.hdf5
+    fps: 10
+    max_steps: 400
+    n_action_steps: 8
+    n_envs: 28
+    n_obs_steps: 2
+    n_test: 50
+    n_test_vis: 4
+    n_train: 6
+    n_train_vis: 2
+    past_action: false
+    render_obs_key: agentview_image
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    test_start_seed: 100000
+    tqdm_interval_sec: 1.0
+    train_start_idx: 0
+  name: can_image
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  task_name: can
+task_name: can_image
+training:
+  checkpoint_every: 50
+  debug: false
+  device: cuda:0
+  gradient_accumulate_every: 1
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  max_train_steps: null
+  max_val_steps: null
+  num_epochs: 1501
+  resume: true
+  rollout_every: 50
+  sample_every: 5
+  seed: 42
+  tqdm_interval_sec: 1.0
+  use_ema: true
+  val_every: 1
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: false

Can/metrics/logs.json.txt

@@ -0,0 +1,31 @@
+{"global_step": 322, "epoch": 0, "max/test_mean_score": 0.0, "k_around_max/test_mean_score": 0.0, "max_k_window/test_mean_score": 0.0, "min_train_loss/test_mean_score": 0.0, "min_val_loss/test_mean_score": 0.0, "k_min_train_loss/test_mean_score": 0.0, "k_min_val_loss/test_mean_score": 0.0, "last/test_mean_score": 0.0, "metric_global_step/test_mean_score": 0}
+{"global_step": 16472, "epoch": 50, "max/test_mean_score": 0.9466666666666667, "k_around_max/test_mean_score": 0.47333333333333333, "max_k_window/test_mean_score": 0.47333333333333333, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.47333333333333333, "k_min_val_loss/test_mean_score": 0.47333333333333333, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 1}
+{"global_step": 32622, "epoch": 100, "max/test_mean_score": 0.9733333333333333, "k_around_max/test_mean_score": 0.6377777777777777, "max_k_window/test_mean_score": 0.578, "min_train_loss/test_mean_score": 0.9666666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.6377777777777777, "k_min_val_loss/test_mean_score": 0.6377777777777777, "last/test_mean_score": 0.9666666666666667, "metric_global_step/test_mean_score": 2}
+{"global_step": 48772, "epoch": 150, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.7183333333333334, "max_k_window/test_mean_score": 0.6713333333333332, "min_train_loss/test_mean_score": 0.96, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.7183333333333333, "k_min_val_loss/test_mean_score": 0.7183333333333334, "last/test_mean_score": 0.96, "metric_global_step/test_mean_score": 3}
+{"global_step": 64922, "epoch": 200, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.7706666666666667, "max_k_window/test_mean_score": 0.7773333333333333, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.7706666666666667, "k_min_val_loss/test_mean_score": 0.7706666666666667, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 4}
+{"global_step": 81072, "epoch": 250, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.8044444444444445, "max_k_window/test_mean_score": 0.872, "min_train_loss/test_mean_score": 0.9733333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.8044444444444445, "k_min_val_loss/test_mean_score": 0.8044444444444445, "last/test_mean_score": 0.9733333333333333, "metric_global_step/test_mean_score": 5}
+{"global_step": 97222, "epoch": 300, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.8211111111111111, "max_k_window/test_mean_score": 0.9726666666666667, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.8295238095238094, "k_min_val_loss/test_mean_score": 0.8295238095238094, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 6}
+{"global_step": 113372, "epoch": 350, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.8277777777777778, "max_k_window/test_mean_score": 0.9766666666666667, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.8483333333333333, "k_min_val_loss/test_mean_score": 0.8483333333333333, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 7}
+{"global_step": 129522, "epoch": 400, "max/test_mean_score": 0.9933333333333333, "k_around_max/test_mean_score": 0.8337037037037037, "max_k_window/test_mean_score": 0.984, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.8644444444444445, "k_min_val_loss/test_mean_score": 0.8644444444444445, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 8}
+{"global_step": 145672, "epoch": 450, "max/test_mean_score": 0.9933333333333333, "k_around_max/test_mean_score": 0.8384444444444444, "max_k_window/test_mean_score": 0.984, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.8766666666666666, "k_min_val_loss/test_mean_score": 0.8766666666666666, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 9}
+{"global_step": 161822, "epoch": 500, "max/test_mean_score": 0.9933333333333333, "k_around_max/test_mean_score": 0.8711111111111111, "max_k_window/test_mean_score": 0.9813333333333332, "min_train_loss/test_mean_score": 0.9733333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9739999999999999, "k_min_val_loss/test_mean_score": 0.9739999999999999, "last/test_mean_score": 0.9733333333333333, "metric_global_step/test_mean_score": 10}
+{"global_step": 177972, "epoch": 550, "max/test_mean_score": 0.9933333333333333, "k_around_max/test_mean_score": 0.8737777777777778, "max_k_window/test_mean_score": 0.9813333333333333, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9773333333333333, "k_min_val_loss/test_mean_score": 0.9733333333333333, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 11}
+{"global_step": 194122, "epoch": 600, "max/test_mean_score": 0.9933333333333333, "k_around_max/test_mean_score": 0.8744444444444445, "max_k_window/test_mean_score": 0.9833333333333331, "min_train_loss/test_mean_score": 0.9733333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9786666666666667, "k_min_val_loss/test_mean_score": 0.9733333333333333, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 12}
+{"global_step": 210272, "epoch": 650, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9306666666666668, "max_k_window/test_mean_score": 0.9879999999999999, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9820000000000001, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 13}
+{"global_step": 226422, "epoch": 700, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9826666666666665, "min_train_loss/test_mean_score": 0.9666666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9806666666666667, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9666666666666667, "metric_global_step/test_mean_score": 14}
+{"global_step": 242572, "epoch": 750, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9853333333333332, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9819999999999999, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 15}
+{"global_step": 258722, "epoch": 800, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9846666666666666, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9819999999999999, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.98, "metric_global_step/test_mean_score": 16}
+{"global_step": 274872, "epoch": 850, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9859999999999998, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9826666666666667, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 17}
+{"global_step": 291022, "epoch": 900, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9853333333333332, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9819999999999999, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 18}
+{"global_step": 307172, "epoch": 950, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9853333333333332, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9820000000000001, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 19}
+{"global_step": 323322, "epoch": 1000, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9893333333333331, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.984, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 20}
+{"global_step": 339472, "epoch": 1050, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9886666666666665, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.984, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 21}
+{"global_step": 355622, "epoch": 1100, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9906666666666665, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9853333333333333, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 22}
+{"global_step": 371772, "epoch": 1150, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9886666666666665, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9853333333333333, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 23}
+{"global_step": 387922, "epoch": 1200, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9899999999999998, "min_train_loss/test_mean_score": 0.98, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9873333333333333, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 24}
+{"global_step": 404072, "epoch": 1250, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9906666666666664, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9873333333333333, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 25}
+{"global_step": 420222, "epoch": 1300, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9886666666666665, "min_train_loss/test_mean_score": 0.9866666666666667, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.988, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 26}
+{"global_step": 436372, "epoch": 1350, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9939999999999998, "min_train_loss/test_mean_score": 1.0, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9893333333333333, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 1.0, "metric_global_step/test_mean_score": 27}
+{"global_step": 452522, "epoch": 1400, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9913333333333331, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9900000000000001, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 28}
+{"global_step": 468672, "epoch": 1450, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9906666666666664, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9900000000000001, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9866666666666667, "metric_global_step/test_mean_score": 29}
+{"global_step": 484822, "epoch": 1500, "max/test_mean_score": 1.0, "k_around_max/test_mean_score": 0.9293333333333332, "max_k_window/test_mean_score": 0.9919999999999997, "min_train_loss/test_mean_score": 0.9933333333333333, "min_val_loss/test_mean_score": 0.9466666666666667, "k_min_train_loss/test_mean_score": 0.9900000000000001, "k_min_val_loss/test_mean_score": 0.9753333333333334, "last/test_mean_score": 0.9933333333333333, "metric_global_step/test_mean_score": 30}

Can/metrics/metrics.json

@@ -0,0 +1,13 @@
+{
+  "epoch": 1500,
+  "global_step": 484822,
+  "k_around_max/test_mean_score": 0.9293333333333332,
+  "k_min_train_loss/test_mean_score": 0.9900000000000001,
+  "k_min_val_loss/test_mean_score": 0.9753333333333334,
+  "last/test_mean_score": 0.9933333333333333,
+  "max/test_mean_score": 1.0,
+  "max_k_window/test_mean_score": 0.9919999999999997,
+  "metric_global_step/test_mean_score": 30,
+  "min_train_loss/test_mean_score": 0.9933333333333333,
+  "min_val_loss/test_mean_score": 0.9466666666666667
+}

Can/metrics/metrics.log

@@ -0,0 +1,39 @@
+2024-11-21 02:19:01,019 [INFO] Monitor waiting for log files!
+2024-11-21 02:19:32,197 [INFO] All log files ready!
+2024-11-21 02:24:46,900 [INFO] Metrics logged at step 322
+2024-11-21 02:56:15,960 [INFO] Metrics logged at step 16472
+2024-11-21 03:28:16,378 [INFO] Metrics logged at step 32622
+2024-11-21 03:59:17,236 [INFO] Metrics logged at step 48772
+2024-11-21 04:30:10,599 [INFO] Metrics logged at step 64922
+2024-11-21 05:01:37,436 [INFO] Metrics logged at step 81072
+2024-11-21 05:32:25,590 [INFO] Metrics logged at step 97222
+2024-11-21 06:03:16,325 [INFO] Metrics logged at step 113372
+2024-11-21 06:33:48,132 [INFO] Metrics logged at step 129522
+2024-11-21 07:04:42,775 [INFO] Metrics logged at step 145672
+2024-11-21 07:35:29,362 [INFO] Metrics logged at step 161822
+2024-11-21 08:06:21,843 [INFO] Metrics logged at step 177972
+2024-11-21 08:37:01,951 [INFO] Metrics logged at step 194122
+2024-11-21 09:07:46,775 [INFO] Metrics logged at step 210272
+2024-11-21 09:38:12,045 [INFO] Metrics logged at step 226422
+2024-11-21 10:08:39,978 [INFO] Metrics logged at step 242572
+2024-11-21 10:38:59,884 [INFO] Metrics logged at step 258722
+2024-11-21 11:09:21,819 [INFO] Metrics logged at step 274872
+2024-11-21 11:39:37,172 [INFO] Metrics logged at step 291022
+2024-11-21 12:09:54,941 [INFO] Metrics logged at step 307172
+2024-11-21 12:40:04,088 [INFO] Metrics logged at step 323322
+2024-11-21 13:10:26,233 [INFO] Metrics logged at step 339472
+2024-11-21 13:40:50,985 [INFO] Metrics logged at step 355622
+2024-11-21 14:11:16,650 [INFO] Metrics logged at step 371772
+2024-11-21 14:41:36,604 [INFO] Metrics logged at step 387922
+2024-11-21 15:11:47,323 [INFO] Metrics logged at step 404072
+2024-11-21 15:42:01,223 [INFO] Metrics logged at step 420222
+2024-11-21 16:12:17,520 [INFO] Metrics logged at step 436372
+2024-11-21 16:42:34,374 [INFO] Metrics logged at step 452522
+2024-11-21 17:12:58,442 [INFO] Metrics logged at step 468672
+2024-11-21 17:42:46,129 [INFO] Metrics logged at step 484822
+2024-11-21 17:42:59,470 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 17:42:59,487 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 17:42:59,504 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 17:42:59,537 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 17:42:59,554 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 17:42:59,570 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Can/train_0/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 42
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_0
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 9zqt69sy_train_0
+  group: 9zqt69sy
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: can_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: can
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4300000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Can/train_0/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_0
+    - hydra.mode=RUN
+    task:
+    - task=can_image_abs
+    - training.seed=42
+    - task.env_runner.test_start_seed=4300000
+    - logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_0
+    - logging.id=9zqt69sy_train_0
+    - logging.group=9zqt69sy
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=9zqt69sy,logging.id=9zqt69sy_train_0,logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_0,task.env_runner.test_start_seed=4300000,task=can_image_abs,training.seed=42
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_0
+    choices:
+      task: can_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Can/train_0/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=can_image_abs
+- training.seed=42
+- task.env_runner.test_start_seed=4300000
+- logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_0
+- logging.id=9zqt69sy_train_0
+- logging.group=9zqt69sy

Can/train_0/checkpoints/epoch=0400-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c36447cb014738d68384e4d1f3ba8de7ec80d7e37f61617fe3aa59bab5635b23
+size 4647697621

Can/train_0/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd1365ba67feedd2a20a67bb1a0060142d851326c948c63cce91056a8503c407
+size 50511031

Can/train_0/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 02:19:09,118][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,624][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,624][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,627][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,627][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,627][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,627][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,628][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,628][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,628][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,632][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,633][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,634][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,635][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,636][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,649][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,650][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,659][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,660][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,776][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,777][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,786][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,789][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,816][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,816][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,961][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,961][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,052][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,053][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,069][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,070][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,074][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,074][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,083][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,083][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,084][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,084][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,094][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,094][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,094][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,094][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,120][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,120][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,121][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,121][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,121][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,122][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,126][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,127][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,129][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,129][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,131][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,131][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,132][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,133][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,157][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,157][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,162][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,163][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,163][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,164][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,165][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,165][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,179][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,180][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,184][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,185][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,189][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,197][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,198][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,214][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,215][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,220][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,221][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,253][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,254][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,277][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,278][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,278][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,279][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,279][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,280][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,398][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,398][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,480][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,480][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,490][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,490][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,497][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,498][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,503][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,503][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,521][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,521][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,522][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,522][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,533][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,533][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,541][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,542][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,558][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,558][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,562][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,562][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,570][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,570][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,590][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,590][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,619][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,619][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,630][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,631][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,641][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,642][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,648][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,649][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,658][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,658][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,668][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,668][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,675][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,676][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,682][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,682][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,697][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,697][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,709][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,709][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,735][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,735][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,753][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,753][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,755][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,755][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,760][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,760][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,767][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,768][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,769][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,770][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,811][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,811][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,813][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,813][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,824][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,825][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,825][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,826][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,831][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,831][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,862][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,862][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,872][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,872][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,880][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,881][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,882][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,882][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,886][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,887][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,888][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,889][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,905][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,905][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,940][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,940][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,945][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,945][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,975][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,976][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,980][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,980][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,995][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,995][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,999][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,999][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,001][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,002][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,005][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,005][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,019][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,019][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,081][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,082][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,085][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,085][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,088][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,089][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,108][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,108][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,110][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,110][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,114][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,115][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,117][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,117][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,122][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,123][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,381][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,381][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,391][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,391][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,552][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,552][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,552][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,552][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,552][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,554][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,591][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,592][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 17:42:49,599][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:42:49,626][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:42:49,642][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:42:49,674][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:42:49,691][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:42:49,706][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Can/train_1/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 43
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_1
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 9zqt69sy_train_1
+  group: 9zqt69sy
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: can_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: can
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4400000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Can/train_1/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_1
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_1
+    - hydra.mode=RUN
+    task:
+    - task=can_image_abs
+    - training.seed=43
+    - task.env_runner.test_start_seed=4400000
+    - logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_1
+    - logging.id=9zqt69sy_train_1
+    - logging.group=9zqt69sy
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=9zqt69sy,logging.id=9zqt69sy_train_1,logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_1,task.env_runner.test_start_seed=4400000,task=can_image_abs,training.seed=43
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_1
+    choices:
+      task: can_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Can/train_1/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=can_image_abs
+- training.seed=43
+- task.env_runner.test_start_seed=4400000
+- logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_1
+- logging.id=9zqt69sy_train_1
+- logging.group=9zqt69sy

Can/train_1/checkpoints/epoch=0100-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f23be863cb6117ec819687b5abc076d7fe66bc79730fe9590dafb1eb99c2624
+size 4647697621

Can/train_1/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d875fb168e929b24d0351577e7beacfe1869fb74daa46540520bb962439cb982
+size 50512012

Can/train_1/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 02:19:09,208][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 02:19:20,621][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,623][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,624][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,627][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,624][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,627][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,628][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,629][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,629][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,629][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,630][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,630][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,630][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,630][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,631][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,632][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,630][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,633][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,632][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,633][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,634][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,636][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,637][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,667][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,667][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,694][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,696][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,698][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,700][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,790][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,792][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,790][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,792][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,939][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,939][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,963][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,963][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,000][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,001][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,094][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,094][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,097][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,097][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,098][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,098][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,100][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,101][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,115][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,115][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,120][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,120][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,121][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,121][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,126][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,127][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,131][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,131][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,156][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,156][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,156][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,157][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,160][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,161][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,162][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,162][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,164][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,164][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,181][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,181][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,181][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,181][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,196][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,197][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,197][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,197][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,203][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,204][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,203][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,204][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,211][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,211][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,215][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,215][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,234][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,234][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,236][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,237][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,255][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,256][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,262][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,262][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,264][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,264][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,361][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,361][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,494][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,494][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,514][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,515][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,518][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,518][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,548][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,548][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,553][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,553][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,565][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,565][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,571][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,571][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,574][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,574][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,581][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,581][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,581][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,581][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,586][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,586][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,588][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,588][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,603][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,603][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,612][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,612][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,648][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,648][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,666][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,666][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,672][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,672][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,684][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,684][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,688][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,688][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,690][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,690][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,704][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,704][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,713][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,713][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,721][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,721][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,723][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,724][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,733][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,733][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,751][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,752][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,772][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,773][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,788][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,788][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,791][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,792][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,833][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,834][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,849][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,849][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,850][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,850][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,892][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,892][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,897][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,897][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,912][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,912][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,912][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,912][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,930][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,930][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,931][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,931][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,934][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,934][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,935][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,935][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,941][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,941][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,969][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,969][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,984][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,985][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,988][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,989][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,013][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,013][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,024][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,024][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,040][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,040][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,058][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,058][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,067][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,068][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,082][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,083][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,086][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,087][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,087][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,088][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,087][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,088][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,098][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,099][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,101][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,103][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,113][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,114][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,294][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,294][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,418][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,418][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,433][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,433][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,448][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,448][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,496][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,496][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,602][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,602][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 17:39:00,549][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:39:00,579][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:39:00,595][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:39:00,630][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:39:00,646][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:39:00,663][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Can/train_2/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 44
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_2
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 9zqt69sy_train_2
+  group: 9zqt69sy
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: can_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: can
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4500000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Can/train_2/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_2
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_2
+    - hydra.mode=RUN
+    task:
+    - task=can_image_abs
+    - training.seed=44
+    - task.env_runner.test_start_seed=4500000
+    - logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_2
+    - logging.id=9zqt69sy_train_2
+    - logging.group=9zqt69sy
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=9zqt69sy,logging.id=9zqt69sy_train_2,logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_2,task.env_runner.test_start_seed=4500000,task=can_image_abs,training.seed=44
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_02-18-13_222588_3043576/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/02.18.38_train_tedi_unet_hybrid_can_image/train_2
+    choices:
+      task: can_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Can/train_2/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=can_image_abs
+- training.seed=44
+- task.env_runner.test_start_seed=4500000
+- logging.name=2024.11.21-02.18.38_train_tedi_unet_hybrid_can_image_train_2
+- logging.id=9zqt69sy_train_2
+- logging.group=9zqt69sy

Can/train_2/checkpoints/epoch=0650-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:accacdec9098e79a2286464e03cd9344be12ddd34ce6184c202fde2d50892ea2
+size 4647697621

Can/train_2/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4be60f42cd44953812a0f6c0c6946ffafa163bef19135fed27d81dbe4bb4e25
+size 50513818

Can/train_2/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 02:19:09,126][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 02:19:20,622][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,623][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,622][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,623][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,623][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,624][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,626][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,632][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,633][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,633][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,634][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,634][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,635][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,642][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,643][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,644][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,644][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,649][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,650][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,657][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,658][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,662][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,664][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,692][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,693][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,715][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,716][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,733][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,736][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,803][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,803][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,855][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,856][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,932][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,933][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,976][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,977][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:20,991][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,992][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:20,998][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:20,999][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,034][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,035][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,037][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,038][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,041][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,041][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,042][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,042][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,055][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,056][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,072][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,072][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,073][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,074][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,079][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,080][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,084][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,084][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,093][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,093][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,099][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,099][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,101][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,101][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,120][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,121][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,121][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,122][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,122][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,122][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,122][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,123][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,132][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,133][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,155][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,155][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,163][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,164][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,180][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,180][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,188][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,189][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,189][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,200][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,201][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,221][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,221][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 02:19:21,263][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,264][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,277][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,277][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,292][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,292][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,325][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,326][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,364][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,364][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,367][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,367][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,375][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,375][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,416][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,416][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,417][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,417][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,427][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,429][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,429][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,468][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,468][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,480][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,481][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,505][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,505][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,509][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,509][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,526][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,526][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,537][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,537][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,546][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,546][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,547][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,547][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,555][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,555][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,576][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,577][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,598][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,598][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,599][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,599][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,609][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,609][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,625][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,625][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 02:19:21,632][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,633][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,658][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,658][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,662][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,662][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,701][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,701][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,715][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,716][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,725][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,726][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,730][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,730][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,735][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,735][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,740][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,740][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,744][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,744][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,780][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,780][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,784][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,784][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,800][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,801][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,805][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,806][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,817][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,817][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,818][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,819][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,845][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,845][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,863][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,863][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,872][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,872][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,888][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,888][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,895][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,896][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,899][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,900][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,911][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,911][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,920][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,921][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,945][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,945][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,948][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,949][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,953][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,954][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:21,957][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,957][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:21,960][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 02:19:21,960][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 02:19:22,026][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,026][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,033][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,033][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,050][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,051][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,079][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,079][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,094][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,095][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,219][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,220][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,251][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,251][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,324][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,324][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 02:19:22,336][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 02:19:22,336][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 17:30:12,258][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:30:12,337][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:30:12,354][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:30:12,389][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:30:12,406][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 17:30:12,422][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/