Upload folder using huggingface_hub

.gitattributes

@@ -52,3 +52,6 @@ Lift/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 Can/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 Can/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 Can/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Square/train_0/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Square/train_1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+Square/train_2/logs.json.txt filter=lfs diff=lfs merge=lfs -text

Square/config.yaml

@@ -0,0 +1,280 @@
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+checkpoint:
+  save_last_ckpt: false
+  save_last_snapshot: false
+  topk:
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+    k: 1
+    mode: max
+    monitor_key: test_mean_score
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: true
+dataset_obs_steps: 2
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  inv_gamma: 1.0
+  max_value: 0.9999
+  min_value: 0.0
+  power: 0.75
+  update_after_step: 0
+exp_name: default
+horizon: 20
+keypoint_visible_rate: 1.0
+logging:
+  group: null
+  id: null
+  mode: online
+  name: 2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image
+  project: diffusion_policy_debug
+  resume: true
+  tags:
+  - train_tedi_unet_hybrid
+  - square_image
+  - default
+multi_run:
+  run_dir: data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image
+  wandb_name_base: 2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image
+n_action_steps: 8
+n_latency_steps: 0
+n_obs_steps: 2
+name: train_tedi_unet_hybrid
+obs_as_global_cond: true
+optimizer:
+  _target_: torch.optim.AdamW
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  lr: 0.0001
+  weight_decay: 1.0e-06
+past_action_visible: false
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  buffer_init: zero
+  chunk_wise_weight: 0.8
+  cond_predict_scale: true
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  eval_fixed_crop: true
+  horizon: 20
+  kernel_size: 5
+  n_action_steps: 8
+  n_groups: 8
+  n_obs_steps: 2
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    beta_start: 0.0001
+    clip_sample: true
+    num_train_timesteps: 100
+    prediction_type: epsilon
+    variance_type: fixed_small
+  num_inference_steps: 100
+  obs_as_global_cond: true
+  obs_encoder_group_norm: true
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+shape_meta:
+  action:
+    shape:
+    - 10
+  obs:
+    agentview_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_eef_pos:
+      shape:
+      - 3
+    robot0_eef_quat:
+      shape:
+      - 4
+    robot0_eye_in_hand_image:
+      shape:
+      - 3
+      - 84
+      - 84
+      type: rgb
+    robot0_gripper_qpos:
+      shape:
+      - 2
+task:
+  abs_action: true
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    abs_action: true
+    dataset_path: data/robomimic/datasets/square/ph/image_abs.hdf5
+    horizon: 20
+    n_obs_steps: 2
+    pad_after: 7
+    pad_before: 1
+    rotation_rep: rotation_6d
+    seed: 42
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    use_cache: true
+    use_legacy_normalizer: false
+    val_ratio: 0.02
+  dataset_path: data/robomimic/datasets/square/ph/image_abs.hdf5
+  dataset_type: ph
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    abs_action: true
+    crf: 22
+    dataset_path: data/robomimic/datasets/square/ph/image_abs.hdf5
+    fps: 10
+    max_steps: 400
+    n_action_steps: 8
+    n_envs: 28
+    n_obs_steps: 2
+    n_test: 50
+    n_test_vis: 4
+    n_train: 6
+    n_train_vis: 2
+    past_action: false
+    render_obs_key: agentview_image
+    shape_meta:
+      action:
+        shape:
+        - 10
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_gripper_qpos:
+          shape:
+          - 2
+    test_start_seed: 100000
+    tqdm_interval_sec: 1.0
+    train_start_idx: 0
+  name: square_image
+  shape_meta:
+    action:
+      shape:
+      - 10
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_gripper_qpos:
+        shape:
+        - 2
+  task_name: square
+task_name: square_image
+training:
+  checkpoint_every: 50
+  debug: false
+  device: cuda:0
+  gradient_accumulate_every: 1
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  max_train_steps: null
+  max_val_steps: null
+  num_epochs: 1501
+  resume: true
+  rollout_every: 50
+  sample_every: 5
+  seed: 42
+  tqdm_interval_sec: 1.0
+  use_ema: true
+  val_every: 1
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  persistent_workers: true
+  pin_memory: true
+  shuffle: false

Square/metrics/logs.json.txt

@@ -0,0 +1,31 @@
+{"global_step": 427, "epoch": 0, "max/test_mean_score": 0.0, "k_around_max/test_mean_score": 0.0, "max_k_window/test_mean_score": 0.0, "min_train_loss/test_mean_score": 0.0, "min_val_loss/test_mean_score": 0.0, "k_min_train_loss/test_mean_score": 0.0, "k_min_val_loss/test_mean_score": 0.0, "last/test_mean_score": 0.0, "metric_global_step/test_mean_score": 0}
+{"global_step": 21827, "epoch": 50, "max/test_mean_score": 0.8666666666666667, "k_around_max/test_mean_score": 0.43333333333333335, "max_k_window/test_mean_score": 0.43333333333333335, "min_train_loss/test_mean_score": 0.8666666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.43333333333333335, "k_min_val_loss/test_mean_score": 0.43333333333333335, "last/test_mean_score": 0.8666666666666667, "metric_global_step/test_mean_score": 1}
+{"global_step": 43227, "epoch": 100, "max/test_mean_score": 0.9400000000000001, "k_around_max/test_mean_score": 0.6022222222222222, "max_k_window/test_mean_score": 0.5566666666666666, "min_train_loss/test_mean_score": 0.9400000000000001, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.6022222222222222, "k_min_val_loss/test_mean_score": 0.6022222222222222, "last/test_mean_score": 0.9400000000000001, "metric_global_step/test_mean_score": 2}
+{"global_step": 64627, "epoch": 150, "max/test_mean_score": 0.9533333333333333, "k_around_max/test_mean_score": 0.69, "max_k_window/test_mean_score": 0.6573333333333333, "min_train_loss/test_mean_score": 0.9533333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.69, "k_min_val_loss/test_mean_score": 0.69, "last/test_mean_score": 0.9533333333333333, "metric_global_step/test_mean_score": 3}
+{"global_step": 86027, "epoch": 200, "max/test_mean_score": 0.96, "k_around_max/test_mean_score": 0.7360000000000001, "max_k_window/test_mean_score": 0.7360000000000001, "min_train_loss/test_mean_score": 0.92, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.7360000000000001, "k_min_val_loss/test_mean_score": 0.7360000000000001, "last/test_mean_score": 0.92, "metric_global_step/test_mean_score": 4}
+{"global_step": 107427, "epoch": 250, "max/test_mean_score": 0.9666666666666667, "k_around_max/test_mean_score": 0.7644444444444445, "max_k_window/test_mean_score": 0.8213333333333335, "min_train_loss/test_mean_score": 0.9066666666666666, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.7644444444444445, "k_min_val_loss/test_mean_score": 0.7644444444444445, "last/test_mean_score": 0.9066666666666666, "metric_global_step/test_mean_score": 5}
+{"global_step": 128827, "epoch": 300, "max/test_mean_score": 0.9733333333333333, "k_around_max/test_mean_score": 0.7904761904761904, "max_k_window/test_mean_score": 0.932, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.7904761904761904, "k_min_val_loss/test_mean_score": 0.7904761904761904, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 6}
+{"global_step": 150227, "epoch": 350, "max/test_mean_score": 0.9733333333333333, "k_around_max/test_mean_score": 0.8083333333333332, "max_k_window/test_mean_score": 0.9333333333333332, "min_train_loss/test_mean_score": 0.9333333333333332, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.8083333333333332, "k_min_val_loss/test_mean_score": 0.8083333333333335, "last/test_mean_score": 0.9333333333333332, "metric_global_step/test_mean_score": 7}
+{"global_step": 171627, "epoch": 400, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8206481481481481, "max_k_window/test_mean_score": 0.9466666666666667, "min_train_loss/test_mean_score": 0.96, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.8251851851851852, "k_min_val_loss/test_mean_score": 0.8251851851851852, "last/test_mean_score": 0.96, "metric_global_step/test_mean_score": 8}
+{"global_step": 193027, "epoch": 450, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8291666666666666, "max_k_window/test_mean_score": 0.944, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.8373333333333334, "k_min_val_loss/test_mean_score": 0.8373333333333335, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 9}
+{"global_step": 214427, "epoch": 500, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8925, "max_k_window/test_mean_score": 0.9393333333333332, "min_train_loss/test_mean_score": 0.9333333333333332, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9306666666666666, "k_min_val_loss/test_mean_score": 0.9306666666666666, "last/test_mean_score": 0.9333333333333332, "metric_global_step/test_mean_score": 10}
+{"global_step": 235827, "epoch": 550, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8978333333333333, "max_k_window/test_mean_score": 0.9493333333333333, "min_train_loss/test_mean_score": 0.9533333333333335, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9393333333333334, "k_min_val_loss/test_mean_score": 0.9306666666666666, "last/test_mean_score": 0.9533333333333335, "metric_global_step/test_mean_score": 11}
+{"global_step": 257227, "epoch": 600, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8998333333333334, "max_k_window/test_mean_score": 0.9566666666666666, "min_train_loss/test_mean_score": 0.9666666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9420000000000001, "k_min_val_loss/test_mean_score": 0.9306666666666666, "last/test_mean_score": 0.9666666666666667, "metric_global_step/test_mean_score": 12}
+{"global_step": 278627, "epoch": 650, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8998333333333334, "max_k_window/test_mean_score": 0.9453333333333332, "min_train_loss/test_mean_score": 0.9333333333333332, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.94, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9333333333333332, "metric_global_step/test_mean_score": 13}
+{"global_step": 300027, "epoch": 700, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8998333333333334, "max_k_window/test_mean_score": 0.9526666666666666, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9426666666666667, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 14}
+{"global_step": 321427, "epoch": 750, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8998333333333334, "max_k_window/test_mean_score": 0.9553333333333333, "min_train_loss/test_mean_score": 0.96, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.948, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.96, "metric_global_step/test_mean_score": 15}
+{"global_step": 342827, "epoch": 800, "max/test_mean_score": 0.98, "k_around_max/test_mean_score": 0.8998333333333334, "max_k_window/test_mean_score": 0.9533333333333331, "min_train_loss/test_mean_score": 0.9533333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9486666666666667, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9533333333333333, "metric_global_step/test_mean_score": 16}
+{"global_step": 364227, "epoch": 850, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9051666666666667, "max_k_window/test_mean_score": 0.9626666666666667, "min_train_loss/test_mean_score": 0.9733333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9526666666666667, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9733333333333333, "metric_global_step/test_mean_score": 17}
+{"global_step": 385627, "epoch": 900, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9051666666666667, "max_k_window/test_mean_score": 0.9633333333333333, "min_train_loss/test_mean_score": 0.9733333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9539999999999998, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9733333333333333, "metric_global_step/test_mean_score": 18}
+{"global_step": 407027, "epoch": 950, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9065, "max_k_window/test_mean_score": 0.9573333333333333, "min_train_loss/test_mean_score": 0.9266666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.952, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9266666666666667, "metric_global_step/test_mean_score": 19}
+{"global_step": 428427, "epoch": 1000, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9065, "max_k_window/test_mean_score": 0.956, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9533333333333331, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 20}
+{"global_step": 449827, "epoch": 1050, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.956, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9526666666666666, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 21}
+{"global_step": 471227, "epoch": 1100, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.956, "min_train_loss/test_mean_score": 0.9533333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9513333333333334, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9533333333333333, "metric_global_step/test_mean_score": 22}
+{"global_step": 492627, "epoch": 1150, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9599999999999999, "min_train_loss/test_mean_score": 0.9666666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9546666666666668, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9666666666666667, "metric_global_step/test_mean_score": 23}
+{"global_step": 514027, "epoch": 1200, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9619999999999999, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9546666666666664, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 24}
+{"global_step": 535427, "epoch": 1250, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9579999999999999, "min_train_loss/test_mean_score": 0.9466666666666667, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9533333333333333, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9466666666666667, "metric_global_step/test_mean_score": 25}
+{"global_step": 556827, "epoch": 1300, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9613333333333332, "min_train_loss/test_mean_score": 0.9333333333333332, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9513333333333334, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9333333333333332, "metric_global_step/test_mean_score": 26}
+{"global_step": 578227, "epoch": 1350, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9586666666666664, "min_train_loss/test_mean_score": 0.9333333333333332, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9473333333333334, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9333333333333332, "metric_global_step/test_mean_score": 27}
+{"global_step": 599627, "epoch": 1400, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9586666666666664, "min_train_loss/test_mean_score": 0.9533333333333333, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9453333333333332, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9533333333333333, "metric_global_step/test_mean_score": 28}
+{"global_step": 621027, "epoch": 1450, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9586666666666664, "min_train_loss/test_mean_score": 0.94, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9453333333333332, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.9266666666666667, "metric_global_step/test_mean_score": 29}
+{"global_step": 642427, "epoch": 1500, "max/test_mean_score": 0.9866666666666667, "k_around_max/test_mean_score": 0.9058333333333334, "max_k_window/test_mean_score": 0.9613333333333332, "min_train_loss/test_mean_score": 0.96, "min_val_loss/test_mean_score": 0.8666666666666667, "k_min_train_loss/test_mean_score": 0.9466666666666667, "k_min_val_loss/test_mean_score": 0.93, "last/test_mean_score": 0.96, "metric_global_step/test_mean_score": 30}

Square/metrics/metrics.json

@@ -0,0 +1,13 @@
+{
+  "epoch": 1500,
+  "global_step": 642427,
+  "k_around_max/test_mean_score": 0.9058333333333334,
+  "k_min_train_loss/test_mean_score": 0.9466666666666667,
+  "k_min_val_loss/test_mean_score": 0.93,
+  "last/test_mean_score": 0.96,
+  "max/test_mean_score": 0.9866666666666667,
+  "max_k_window/test_mean_score": 0.9613333333333332,
+  "metric_global_step/test_mean_score": 30,
+  "min_train_loss/test_mean_score": 0.96,
+  "min_val_loss/test_mean_score": 0.8666666666666667
+}

Square/metrics/metrics.log

@@ -0,0 +1,39 @@
+2024-11-21 04:18:56,386 [INFO] Monitor waiting for log files!
+2024-11-21 04:19:26,579 [INFO] All log files ready!
+2024-11-21 04:24:50,872 [INFO] Metrics logged at step 427
+2024-11-21 05:03:04,798 [INFO] Metrics logged at step 21827
+2024-11-21 05:41:11,534 [INFO] Metrics logged at step 43227
+2024-11-21 06:19:01,975 [INFO] Metrics logged at step 64627
+2024-11-21 06:56:46,023 [INFO] Metrics logged at step 86027
+2024-11-21 07:34:22,612 [INFO] Metrics logged at step 107427
+2024-11-21 08:12:14,052 [INFO] Metrics logged at step 128827
+2024-11-21 08:50:08,774 [INFO] Metrics logged at step 150227
+2024-11-21 09:28:08,120 [INFO] Metrics logged at step 171627
+2024-11-21 10:05:54,204 [INFO] Metrics logged at step 193027
+2024-11-21 10:43:31,233 [INFO] Metrics logged at step 214427
+2024-11-21 11:21:26,046 [INFO] Metrics logged at step 235827
+2024-11-21 11:59:11,006 [INFO] Metrics logged at step 257227
+2024-11-21 12:36:48,865 [INFO] Metrics logged at step 278627
+2024-11-21 13:14:31,153 [INFO] Metrics logged at step 300027
+2024-11-21 13:53:09,466 [INFO] Metrics logged at step 321427
+2024-11-21 14:31:21,745 [INFO] Metrics logged at step 342827
+2024-11-21 15:09:06,115 [INFO] Metrics logged at step 364227
+2024-11-21 15:46:45,102 [INFO] Metrics logged at step 385627
+2024-11-21 16:25:09,630 [INFO] Metrics logged at step 407027
+2024-11-21 17:02:56,687 [INFO] Metrics logged at step 428427
+2024-11-21 17:40:37,776 [INFO] Metrics logged at step 449827
+2024-11-21 18:18:22,405 [INFO] Metrics logged at step 471227
+2024-11-21 18:56:31,072 [INFO] Metrics logged at step 492627
+2024-11-21 19:34:00,942 [INFO] Metrics logged at step 514027
+2024-11-21 20:11:44,329 [INFO] Metrics logged at step 535427
+2024-11-21 20:49:45,499 [INFO] Metrics logged at step 556827
+2024-11-21 21:27:27,745 [INFO] Metrics logged at step 578227
+2024-11-21 22:05:34,666 [INFO] Metrics logged at step 599627
+2024-11-21 22:43:34,194 [INFO] Metrics logged at step 621027
+2024-11-21 23:21:04,493 [INFO] Metrics logged at step 642427
+2024-11-21 23:21:16,035 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 23:21:16,053 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 23:21:16,069 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 23:21:16,103 [WARNING] Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 23:21:16,119 [WARNING] Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+2024-11-21 23:21:16,136 [WARNING] Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Square/train_0/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 42
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_0
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 2hq6hebh_train_0
+  group: 2hq6hebh
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: square_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: square
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4300000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Square/train_0/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_0
+    - hydra.mode=RUN
+    task:
+    - task=square_image_abs
+    - training.seed=42
+    - task.env_runner.test_start_seed=4300000
+    - logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_0
+    - logging.id=2hq6hebh_train_0
+    - logging.group=2hq6hebh
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=2hq6hebh,logging.id=2hq6hebh_train_0,logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_0,task.env_runner.test_start_seed=4300000,task=square_image_abs,training.seed=42
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_0
+    choices:
+      task: square_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Square/train_0/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=square_image_abs
+- training.seed=42
+- task.env_runner.test_start_seed=4300000
+- logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_0
+- logging.id=2hq6hebh_train_0
+- logging.group=2hq6hebh

Square/train_0/checkpoints/epoch=0850-test_mean_score=1.000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:912f70b17501036dbb69228f79b824e5cfc002acc5b2bc9959b06539319522b5
+size 4647697685

Square/train_0/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:887c9aca18f15f2459adaeddb9d3a42f33d75bb6035c10a026f34b577ec9d904
+size 66893869

Square/train_0/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 04:19:04,225][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,429][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,430][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,434][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,435][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,441][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,442][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,448][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,450][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,457][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,457][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,468][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,469][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,481][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,482][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,487][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,488][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,491][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,492][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,496][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,497][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,500][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,501][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,531][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,533][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,564][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,565][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,565][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,565][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,686][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,687][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,697][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,698][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,736][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,737][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,739][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,739][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,739][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,739][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,742][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,742][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,746][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,746][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,747][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,747][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,752][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,752][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,761][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,761][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,763][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,764][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,767][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,768][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,774][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,774][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,777][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,777][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,783][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,783][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,788][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,788][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,805][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,805][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,818][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,819][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,836][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,837][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,837][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,837][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,851][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,855][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,857][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,857][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,893][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,893][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,896][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,896][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,899][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,899][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,903][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,903][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,908][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,908][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,908][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,908][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,963][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,964][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,984][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,985][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,050][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,050][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,078][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,078][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,095][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,096][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,105][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,106][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,107][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,107][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,107][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,107][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,124][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,124][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,134][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,135][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,137][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,137][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,140][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,140][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,143][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,144][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,153][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,153][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,163][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,163][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,166][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,166][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,201][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,201][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,205][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,206][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,214][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,214][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,228][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,229][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,249][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,249][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,250][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,250][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,276][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,277][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,282][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,282][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,292][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,292][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,299][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,300][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,318][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,319][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,324][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,325][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,334][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,334][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,399][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,400][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,484][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,484][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,494][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,494][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,513][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,514][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,521][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,521][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,533][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,533][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,543][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,543][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,547][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,548][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,565][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,566][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,587][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,588][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,593][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,593][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,601][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,601][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,608][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,609][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,610][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,611][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,633][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,633][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,670][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,671][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,681][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,681][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,682][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,683][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,689][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,689][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,710][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,711][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,711][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,711][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,718][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,718][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,721][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,722][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,757][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,757][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,782][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,782][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,783][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,784][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,784][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,784][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,787][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,787][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,793][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,793][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,130][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,130][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,199][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,199][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,268][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,269][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,302][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,303][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 23:21:06,783][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:21:06,856][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:21:06,873][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:21:06,906][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:21:06,923][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:21:06,940][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Square/train_1/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 43
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_1
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 2hq6hebh_train_1
+  group: 2hq6hebh
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: square_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: square
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4400000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Square/train_1/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_1
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_1
+    - hydra.mode=RUN
+    task:
+    - task=square_image_abs
+    - training.seed=43
+    - task.env_runner.test_start_seed=4400000
+    - logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_1
+    - logging.id=2hq6hebh_train_1
+    - logging.group=2hq6hebh
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=2hq6hebh,logging.id=2hq6hebh_train_1,logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_1,task.env_runner.test_start_seed=4400000,task=square_image_abs,training.seed=43
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_1
+    choices:
+      task: square_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Square/train_1/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=square_image_abs
+- training.seed=43
+- task.env_runner.test_start_seed=4400000
+- logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_1
+- logging.id=2hq6hebh_train_1
+- logging.group=2hq6hebh

Square/train_1/checkpoints/epoch=0150-test_mean_score=0.980.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d18532b7d8549a26d893f60d28e6cbb7fb015ca9817f220948643150db0f38c5
+size 4647697621

Square/train_1/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f48eb9fa03e8af26a8e9100dd9e60aca02d89ae9c9f9848488cfbe083c811ca2
+size 66896822

Square/train_1/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 04:19:04,217][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,426][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,428][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,427][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,429][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,429][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,429][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,430][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,431][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,437][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,438][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,444][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,445][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,455][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,455][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,460][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,461][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,463][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,464][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,472][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,473][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,474][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,475][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,479][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,479][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,492][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,493][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,495][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,496][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,501][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,502][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,515][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,515][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,521][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,523][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:15,683][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,683][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,688][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,689][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,690][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,690][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,694][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,694][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,703][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,703][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,706][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,707][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,711][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,712][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,718][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,718][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,724][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,724][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,724][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,725][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,728][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,729][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,729][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,729][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,733][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,733][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,735][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,735][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,750][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,751][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,755][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,756][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,760][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,760][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,772][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,773][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,782][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,783][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,792][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,792][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,794][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,795][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,798][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,798][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,800][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,801][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,814][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,814][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,825][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,825][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,827][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,828][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,832][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,832][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,855][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,855][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:15,925][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,925][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,950][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,950][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,950][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,951][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,957][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,957][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,960][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,960][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,970][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,970][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,971][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,971][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:15,986][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:15,986][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,013][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,013][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,013][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,014][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,022][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,022][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,022][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,023][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,024][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,024][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,041][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,041][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,047][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,047][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,048][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,048][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,070][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,070][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,078][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,078][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,090][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,091][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,097][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,098][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,101][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,102][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,106][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,106][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,111][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,111][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,114][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,115][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,120][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,120][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,133][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,134][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,172][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,172][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,177][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,177][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,192][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,192][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,252][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,253][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,256][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,257][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,258][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,259][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,277][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,277][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,297][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,297][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,300][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,300][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,318][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,318][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,323][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,323][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,332][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,332][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,336][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,336][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,340][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,341][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,352][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,352][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,352][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,353][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,357][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,357][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,359][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,360][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,372][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,373][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,381][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,382][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,383][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,383][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,411][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,411][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,436][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,436][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,449][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,450][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,451][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,452][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,478][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,478][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,498][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,498][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,504][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,504][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,507][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,508][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:16,521][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,521][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 22:47:31,591][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 22:47:31,630][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 22:47:31,646][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 22:47:31,679][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 22:47:31,695][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 22:47:31,712][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/

Square/train_2/.hydra/config.yaml

@@ -0,0 +1,230 @@
+name: train_tedi_unet_hybrid
+_target_: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: default
+horizon: 20
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
+  shape_meta: ${shape_meta}
+  noise_scheduler:
+    _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 100
+  obs_as_global_cond: ${obs_as_global_cond}
+  crop_shape:
+  - 76
+  - 76
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  kernel_size: 5
+  n_groups: 8
+  cond_predict_scale: true
+  obs_encoder_group_norm: true
+  eval_fixed_crop: true
+  temporally_constant_weight: 0.2
+  temporally_increasing_weight: 0.0
+  temporally_random_weights: 0.0
+  chunk_wise_weight: 0.8
+  buffer_init: zero
+ema:
+  _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: true
+  pin_memory: true
+  persistent_workers: true
+val_dataloader:
+  batch_size: 64
+  num_workers: 16
+  shuffle: false
+  pin_memory: true
+  persistent_workers: true
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 44
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 1501
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: diffusion_policy_debug
+  resume: true
+  mode: online
+  name: 2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_2
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: 2hq6hebh_train_2
+  group: 2hq6hebh
+checkpoint:
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: false
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: square_image
+  shape_meta:
+    obs:
+      agentview_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eye_in_hand_image:
+        shape:
+        - 3
+        - 84
+        - 84
+        type: rgb
+      robot0_eef_pos:
+        shape:
+        - 3
+      robot0_eef_quat:
+        shape:
+        - 4
+      robot0_gripper_qpos:
+        shape:
+        - 2
+    action:
+      shape:
+      - 10
+  task_name: square
+  dataset_type: ph
+  dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+  abs_action: true
+  env_runner:
+    _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    n_train: 6
+    n_train_vis: 2
+    train_start_idx: 0
+    n_test: 50
+    n_test_vis: 4
+    test_start_seed: 4500000
+    max_steps: ${eval:'500 if "${task.dataset_type}" == "mh" else 400'}
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    render_obs_key: agentview_image
+    fps: 10
+    crf: 22
+    past_action: ${past_action_visible}
+    abs_action: true
+    tqdm_interval_sec: 1.0
+    n_envs: 28
+  dataset:
+    _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
+    shape_meta:
+      obs:
+        agentview_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eye_in_hand_image:
+          shape:
+          - 3
+          - 84
+          - 84
+          type: rgb
+        robot0_eef_pos:
+          shape:
+          - 3
+        robot0_eef_quat:
+          shape:
+          - 4
+        robot0_gripper_qpos:
+          shape:
+          - 2
+      action:
+        shape:
+        - 10
+    dataset_path: data/robomimic/datasets/${task.task_name}/${task.dataset_type}/image_abs.hdf5
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1+${n_latency_steps}'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    n_obs_steps: ${dataset_obs_steps}
+    abs_action: true
+    rotation_rep: rotation_6d
+    use_legacy_normalizer: false
+    use_cache: true
+    seed: 42
+    val_ratio: 0.02

Square/train_2/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_2
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_2
+    - hydra.mode=RUN
+    task:
+    - task=square_image_abs
+    - training.seed=44
+    - task.env_runner.test_start_seed=4500000
+    - logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_2
+    - logging.id=2hq6hebh_train_2
+    - logging.group=2hq6hebh
+  job:
+    name: train
+    chdir: null
+    override_dirname: logging.group=2hq6hebh,logging.id=2hq6hebh_train_2,logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_2,task.env_runner.test_start_seed=4500000,task=square_image_abs,training.seed=44
+    id: ???
+    num: ???
+    config_name: tedi_robomimic.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: main
+    - path: hydra_plugins.hydra_colorlog.conf
+      schema: pkg
+      provider: hydra-colorlog
+    - path: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/diffusion_policy/config
+      schema: file
+      provider: command-line
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp/ray/session_2024-11-21_04-18-16_775476_3838193/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.21/04.18.34_train_tedi_unet_hybrid_square_image/train_2
+    choices:
+      task: square_image_abs
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

Square/train_2/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- task=square_image_abs
+- training.seed=44
+- task.env_runner.test_start_seed=4500000
+- logging.name=2024.11.21-04.18.34_train_tedi_unet_hybrid_square_image_train_2
+- logging.id=2hq6hebh_train_2
+- logging.group=2hq6hebh

Square/train_2/checkpoints/epoch=0400-test_mean_score=0.980.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc2d37014e16ecc8d91cad710f65846dae555e9503e4a1ebeea19c94ef39adf0
+size 4647697685

Square/train_2/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:443999802628c775656fbce894de7601971ca2157c271dc49084e70e9f9496d2
+size 66897229

Square/train_2/train.log

@@ -0,0 +1,231 @@
+[2024-11-21 04:19:04,213][diffusion_policy.model.diffusion.conditional_unet1d_tedi][INFO] - number of parameters: 2.568572e+08
+[2024-11-21 04:19:16,219][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,220][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,233][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,234][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,284][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,285][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,335][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,337][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,368][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,370][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,424][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,425][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,425][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,426][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,441][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,441][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,472][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,473][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,490][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,491][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,530][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,530][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,548][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,550][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,559][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,559][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,571][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,572][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,604][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,605][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,633][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,634][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,658][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,658][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,675][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,675][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,680][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,681][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,682][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,683][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,712][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,712][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,718][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,719][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,743][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,744][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,747][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,748][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,748][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,749][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,783][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,784][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,795][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,795][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,821][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,822][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,851][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,852][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,904][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,905][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,920][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,922][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,924][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,925][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,929][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,929][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,944][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,944][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,951][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,952][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,956][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,956][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:16,959][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,959][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:16,960][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,960][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:16,986][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:16,987][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,012][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,014][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:17,019][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,020][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,030][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,030][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,067][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,068][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,102][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,102][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,104][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,105][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,123][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,124][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,124][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,125][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:17,130][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,130][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,131][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:17,131][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,139][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '0']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,140][root][INFO] - Device 0 is not available for rendering
+[2024-11-21 04:19:17,168][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,168][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,189][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,190][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,194][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,194][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,198][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,199][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,280][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,280][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,334][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,334][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,338][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,341][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,341][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,342][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,343][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,343][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,368][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,368][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,376][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,376][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,387][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,387][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,410][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,411][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,411][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,411][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,413][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,414][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,423][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,423][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,435][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,435][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,437][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,437][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,459][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,459][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,464][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,464][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,467][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,467][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,496][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,498][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '1']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,499][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,499][root][INFO] - Device 1 is not available for rendering
+[2024-11-21 04:19:17,498][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,499][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,506][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,506][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,512][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,513][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,513][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,514][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,518][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,519][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,522][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,523][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,525][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,525][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,535][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,535][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,547][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,547][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,549][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,550][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,647][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,647][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,649][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,650][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,663][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,664][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,666][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,666][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,715][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,716][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,725][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,726][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,754][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,755][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,774][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,775][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,785][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,786][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,812][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,812][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,817][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,818][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,827][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,828][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,839][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,839][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,879][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,880][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,923][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,924][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,925][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,926][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:17,931][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,932][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,939][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '2']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,940][root][INFO] - Device 2 is not available for rendering
+[2024-11-21 04:19:17,972][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:17,972][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,017][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,018][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,071][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,072][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,118][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,119][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,125][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,126][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,169][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,170][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,239][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,240][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,240][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,240][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,262][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,263][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,266][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,266][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 04:19:18,276][root][INFO] - Command '['/root/mambaforge/envs/robodiff/lib/python3.9/site-packages/egl_probe/build/test_device', '3']' returned non-zero exit status 1.
+[2024-11-21 04:19:18,276][root][INFO] - Device 3 is not available for rendering
+[2024-11-21 23:20:27,637][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:20:27,669][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:20:27,686][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:20:27,719][urllib3.connectionpool][WARNING] - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:20:27,736][urllib3.connectionpool][WARNING] - Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/
+[2024-11-21 23:20:27,753][urllib3.connectionpool][WARNING] - Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'SSLError(FileNotFoundError(2, 'No such file or directory'))': /api/4504800232407040/envelope/