Add files using upload-large-folder tool

adroit_hammer-dp3-1218_seed0/wandb/debug.log

@@ -0,0 +1,24 @@
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_setup.py:_flush():68] Configure stats pid to 4158545
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-19 02:22:47,801 INFO    MainThread:4158545 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/logs/debug.log
+2024-12-19 02:22:47,802 INFO    MainThread:4158545 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/logs/debug-internal.log
+2024-12-19 02:22:47,802 INFO    MainThread:4158545 [wandb_init.py:init():644] calling init triggers
+2024-12-19 02:22:47,802 INFO    MainThread:4158545 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'adroit_hammer', 'task_name': 'hammer', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'hammer', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_hammer_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_hammer', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'exp_name': 'adroit_hammer-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_hammer-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.22.45_train_dp3_adroit_hammer', 'wandb_name_base': '2024.12.19-02.22.45_train_dp3_adroit_hammer'}}
+2024-12-19 02:22:47,802 INFO    MainThread:4158545 [wandb_init.py:init():680] starting backend
+2024-12-19 02:22:47,802 INFO    MainThread:4158545 [wandb_init.py:init():684] sending inform_init request
+2024-12-19 02:22:47,830 INFO    MainThread:4158545 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-19 02:22:47,830 INFO    MainThread:4158545 [wandb_init.py:init():697] backend started and connected
+2024-12-19 02:22:47,837 INFO    MainThread:4158545 [wandb_init.py:init():790] updated telemetry
+2024-12-19 02:22:48,020 INFO    MainThread:4158545 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-19 02:22:49,553 INFO    MainThread:4158545 [wandb_init.py:init():874] starting run threads in backend
+2024-12-19 02:22:49,672 INFO    MainThread:4158545 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-19 02:22:49,673 INFO    MainThread:4158545 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-19 02:22:49,673 INFO    MainThread:4158545 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-19 02:22:49,673 INFO    MainThread:4158545 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-19 02:22:49,675 INFO    MainThread:4158545 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-19 02:22:49,676 INFO    MainThread:4158545 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0'}
+2024-12-19 03:42:27,921 WARNING MsgRouterThr:4158545 [router.py:message_loop():75] message_loop has been closed

adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/config.yaml

@@ -0,0 +1,244 @@
+_wandb:
+    value:
+        cli_version: 0.19.1
+        m: []
+        python_version: 3.8.20
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.8.20
+            "5": 0.19.1
+            "8":
+                - 5
+            "12": 0.19.1
+            "13": linux-x86_64
+checkpoint:
+    value:
+        save_ckpt: true
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+            k: 1
+            mode: max
+            monitor_key: test_mean_score
+dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: true
+dataset_obs_steps:
+    value: 2
+ema:
+    value:
+        _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+        inv_gamma: 1
+        max_value: 0.9999
+        min_value: 0
+        power: 0.75
+        update_after_step: 0
+exp_name:
+    value: adroit_hammer-dp3-1218
+horizon:
+    value: 4
+keypoint_visible_rate:
+    value: 1
+logging:
+    value:
+        group: adroit_hammer-dp3-1218
+        id: null
+        mode: online
+        name: "0"
+        project: dp3
+        resume: true
+        tags:
+            - dp3
+multi_run:
+    value:
+        run_dir: data/outputs/2024.12.19/02.22.45_train_dp3_adroit_hammer
+        wandb_name_base: 2024.12.19-02.22.45_train_dp3_adroit_hammer
+n_action_steps:
+    value: 3
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 2
+name:
+    value: train_dp3
+obs_as_global_cond:
+    value: true
+optimizer:
+    value:
+        _target_: torch.optim.AdamW
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0
+policy:
+    value:
+        _target_: diffusion_policy_3d.policy.dp3.DP3
+        condition_type: film
+        crop_shape:
+            - 80
+            - 80
+        diffusion_step_embed_dim: 128
+        down_dims:
+            - 512
+            - 1024
+            - 2048
+        encoder_output_dim: 64
+        horizon: 4
+        kernel_size: 5
+        n_action_steps: 3
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler:
+            _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+            beta_end: 0.02
+            beta_schedule: squaredcos_cap_v2
+            beta_start: 0.0001
+            clip_sample: true
+            num_train_timesteps: 100
+            prediction_type: sample
+            set_alpha_to_one: true
+            steps_offset: 0
+        num_inference_steps: 10
+        obs_as_global_cond: true
+        pointcloud_encoder_cfg:
+            final_norm: layernorm
+            in_channels: 3
+            normal_channel: false
+            out_channels: 64
+            use_layernorm: true
+        pointnet_type: pointnet
+        shape_meta:
+            action:
+                shape:
+                    - 26
+            obs:
+                agent_pos:
+                    shape:
+                        - 24
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        use_down_condition: true
+        use_mid_condition: true
+        use_pc_color: false
+        use_point_crop: true
+        use_up_condition: true
+shape_meta:
+    value:
+        action:
+            shape:
+                - 26
+        obs:
+            agent_pos:
+                shape:
+                    - 24
+                type: low_dim
+            point_cloud:
+                shape:
+                    - 512
+                    - 3
+                type: point_cloud
+task:
+    value:
+        dataset:
+            _target_: diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset
+            horizon: 4
+            max_train_episodes: 90
+            pad_after: 2
+            pad_before: 1
+            seed: 42
+            val_ratio: 0.02
+            zarr_path: /data/haojun/datasets/3d-dp/adroit_hammer_expert.zarr
+        env_runner:
+            _target_: diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner
+            eval_episodes: 20
+            fps: 10
+            max_steps: 300
+            n_action_steps: 3
+            n_obs_steps: 2
+            render_size: 84
+            task_name: hammer
+            use_point_crop: true
+        image_shape:
+            - 3
+            - 84
+            - 84
+        name: adroit_hammer
+        shape_meta:
+            action:
+                shape:
+                    - 26
+            obs:
+                agent_pos:
+                    shape:
+                        - 24
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        task_name: hammer
+task_name:
+    value: adroit_hammer
+training:
+    value:
+        checkpoint_every: 200
+        debug: false
+        device: cuda:0
+        gradient_accumulate_every: 1
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 3000
+        resume: true
+        rollout_every: 200
+        sample_every: 5
+        seed: 0
+        tqdm_interval_sec: 1
+        use_ema: true
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: false

adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os":  "Linux-5.15.0-117-generic-x86_64-with-glibc2.17",
+  "python":  "CPython 3.8.20",
+  "startedAt":  "2024-12-18T18:22:47.830749Z",
+  "args":  [
+    "--config-name=dp3.yaml",
+    "task=adroit_hammer",
+    "hydra.run.dir=data/outputs/adroit_hammer-dp3-1218_seed0",
+    "training.debug=False",
+    "training.seed=0",
+    "training.device=cuda:0",
+    "exp_name=adroit_hammer-dp3-1218",
+    "logging.mode=online",
+    "checkpoint.save_ckpt=True"
+  ],
+  "program":  "train.py",
+  "codePath":  "3D-Diffusion-Policy/train.py",
+  "git":  {
+    "remote":  "git@github.com:YanjieZe/3D-Diffusion-Policy.git",
+    "commit":  "3153fb32c17308fc828015156480b77f6b2e666a"
+  },
+  "email":  "chenhaojun001@gmail.com",
+  "root":  "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0",
+  "host":  "Server-3090",
+  "executable":  "/home/chenhaojun/miniconda3/envs/dp3/bin/python",
+  "codePathLocal":  "train.py",
+  "cpu_count":  128,
+  "cpu_count_logical":  256,
+  "gpu":  "NVIDIA GeForce RTX 3090",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "943412031488",
+      "used":  "837418799104"
+    }
+  },
+  "memory":  {
+    "total":  "1081980669952"
+  },
+  "cpu":  {
+    "count":  128,
+    "countLogical":  256
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "12.5"
+}

adroit_pen-dp3-1218_seed0/.hydra/hydra.yaml

@@ -0,0 +1,163 @@
+hydra:
+  run:
+    dir: data/outputs/adroit_pen-dp3-1218_seed0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/adroit_pen-dp3-1218_seed0
+    - hydra.mode=RUN
+    task:
+    - task=adroit_pen
+    - training.debug=False
+    - training.seed=0
+    - training.device=cuda:0
+    - exp_name=adroit_pen-dp3-1218
+    - logging.mode=online
+    - checkpoint.save_ckpt=True
+  job:
+    name: train
+    chdir: null
+    override_dirname: checkpoint.save_ckpt=True,exp_name=adroit_pen-dp3-1218,logging.mode=online,task=adroit_pen,training.debug=False,training.device=cuda:0,training.seed=0
+    id: ???
+    num: ???
+    config_name: dp3.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0
+    choices:
+      task: adroit_pen
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

adroit_pen-dp3-1218_seed0/wandb/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-19T02:23:11.067012008+08:00","level":"INFO","msg":"using version","core version":"0.19.1"}
+{"time":"2024-12-19T02:23:11.067039698+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-core.log"}
+{"time":"2024-12-19T02:23:11.287533236+08:00","level":"INFO","msg":"created new stream","id":"wnx82u0h"}
+{"time":"2024-12-19T02:23:11.287579885+08:00","level":"INFO","msg":"stream: started","id":"wnx82u0h"}
+{"time":"2024-12-19T02:23:11.287650415+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T02:23:11.287714795+08:00","level":"INFO","msg":"handler: started","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T02:23:11.287915284+08:00","level":"INFO","msg":"sender: started","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T02:23:13.851169857+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-19T03:39:20.847103676+08:00","level":"INFO","msg":"stream: closing","id":"wnx82u0h"}
+{"time":"2024-12-19T03:39:20.847167426+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-19T03:39:20.848587386+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-19T03:39:23.680955487+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-19T03:39:26.446790779+08:00","level":"INFO","msg":"handler: closed","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T03:39:26.446866839+08:00","level":"INFO","msg":"sender: closed","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T03:39:26.446853389+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"wnx82u0h"}
+{"time":"2024-12-19T03:39:26.447021069+08:00","level":"INFO","msg":"stream: closed","id":"wnx82u0h"}

adroit_pen-dp3-1218_seed0/wandb/debug.log

@@ -0,0 +1,24 @@
+2024-12-19 02:23:11,035 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Configure stats pid to 4160108
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-internal.log
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:init():644] calling init triggers
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'adroit_pen', 'task_name': 'pen', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'pen', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_pen_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_pen', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'exp_name': 'adroit_pen-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_pen-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen', 'wandb_name_base': '2024.12.19-02.23.08_train_dp3_adroit_pen'}}
+2024-12-19 02:23:11,037 INFO    MainThread:4160108 [wandb_init.py:init():680] starting backend
+2024-12-19 02:23:11,037 INFO    MainThread:4160108 [wandb_init.py:init():684] sending inform_init request
+2024-12-19 02:23:11,063 INFO    MainThread:4160108 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-19 02:23:11,064 INFO    MainThread:4160108 [wandb_init.py:init():697] backend started and connected
+2024-12-19 02:23:11,069 INFO    MainThread:4160108 [wandb_init.py:init():790] updated telemetry
+2024-12-19 02:23:11,259 INFO    MainThread:4160108 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-19 02:23:13,842 INFO    MainThread:4160108 [wandb_init.py:init():874] starting run threads in backend
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-19 02:23:13,969 INFO    MainThread:4160108 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-19 02:23:13,969 INFO    MainThread:4160108 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0'}
+2024-12-19 03:39:20,847 WARNING MsgRouterThr:4160108 [router.py:message_loop():75] message_loop has been closed

adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/config.yaml

@@ -0,0 +1,244 @@
+_wandb:
+    value:
+        cli_version: 0.19.1
+        m: []
+        python_version: 3.8.20
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.8.20
+            "5": 0.19.1
+            "8":
+                - 5
+            "12": 0.19.1
+            "13": linux-x86_64
+checkpoint:
+    value:
+        save_ckpt: true
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+            k: 1
+            mode: max
+            monitor_key: test_mean_score
+dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: true
+dataset_obs_steps:
+    value: 2
+ema:
+    value:
+        _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+        inv_gamma: 1
+        max_value: 0.9999
+        min_value: 0
+        power: 0.75
+        update_after_step: 0
+exp_name:
+    value: adroit_pen-dp3-1218
+horizon:
+    value: 4
+keypoint_visible_rate:
+    value: 1
+logging:
+    value:
+        group: adroit_pen-dp3-1218
+        id: null
+        mode: online
+        name: "0"
+        project: dp3
+        resume: true
+        tags:
+            - dp3
+multi_run:
+    value:
+        run_dir: data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen
+        wandb_name_base: 2024.12.19-02.23.08_train_dp3_adroit_pen
+n_action_steps:
+    value: 3
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 2
+name:
+    value: train_dp3
+obs_as_global_cond:
+    value: true
+optimizer:
+    value:
+        _target_: torch.optim.AdamW
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0
+policy:
+    value:
+        _target_: diffusion_policy_3d.policy.dp3.DP3
+        condition_type: film
+        crop_shape:
+            - 80
+            - 80
+        diffusion_step_embed_dim: 128
+        down_dims:
+            - 512
+            - 1024
+            - 2048
+        encoder_output_dim: 64
+        horizon: 4
+        kernel_size: 5
+        n_action_steps: 3
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler:
+            _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+            beta_end: 0.02
+            beta_schedule: squaredcos_cap_v2
+            beta_start: 0.0001
+            clip_sample: true
+            num_train_timesteps: 100
+            prediction_type: sample
+            set_alpha_to_one: true
+            steps_offset: 0
+        num_inference_steps: 10
+        obs_as_global_cond: true
+        pointcloud_encoder_cfg:
+            final_norm: layernorm
+            in_channels: 3
+            normal_channel: false
+            out_channels: 64
+            use_layernorm: true
+        pointnet_type: pointnet
+        shape_meta:
+            action:
+                shape:
+                    - 24
+            obs:
+                agent_pos:
+                    shape:
+                        - 24
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        use_down_condition: true
+        use_mid_condition: true
+        use_pc_color: false
+        use_point_crop: true
+        use_up_condition: true
+shape_meta:
+    value:
+        action:
+            shape:
+                - 24
+        obs:
+            agent_pos:
+                shape:
+                    - 24
+                type: low_dim
+            point_cloud:
+                shape:
+                    - 512
+                    - 3
+                type: point_cloud
+task:
+    value:
+        dataset:
+            _target_: diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset
+            horizon: 4
+            max_train_episodes: 90
+            pad_after: 2
+            pad_before: 1
+            seed: 42
+            val_ratio: 0.02
+            zarr_path: /data/haojun/datasets/3d-dp/adroit_pen_expert.zarr
+        env_runner:
+            _target_: diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner
+            eval_episodes: 20
+            fps: 10
+            max_steps: 300
+            n_action_steps: 3
+            n_obs_steps: 2
+            render_size: 84
+            task_name: pen
+            use_point_crop: true
+        image_shape:
+            - 3
+            - 84
+            - 84
+        name: adroit_pen
+        shape_meta:
+            action:
+                shape:
+                    - 24
+            obs:
+                agent_pos:
+                    shape:
+                        - 24
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        task_name: pen
+task_name:
+    value: adroit_pen
+training:
+    value:
+        checkpoint_every: 200
+        debug: false
+        device: cuda:0
+        gradient_accumulate_every: 1
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 3000
+        resume: true
+        rollout_every: 200
+        sample_every: 5
+        seed: 0
+        tqdm_interval_sec: 1
+        use_ema: true
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: false

adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/output.log

@@ -0,0 +1,16 @@
+                                                                                                                             
+test_mean_score: 0.0
+test_mean_score: 0.35
+test_mean_score: 0.25
+test_mean_score: 0.15
+test_mean_score: 0.5
+test_mean_score: 0.35
+test_mean_score: 0.35
+test_mean_score: 0.3
+test_mean_score: 0.3
+test_mean_score: 0.425
+test_mean_score: 0.5
+test_mean_score: 0.45
+test_mean_score: 0.25
+test_mean_score: 0.25
+test_mean_score: 0.45

adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/requirements.txt

@@ -0,0 +1,175 @@
+diffusion_policy_3d==0.0.0
+certifi==2024.12.14
+PyYAML==6.0.2
+dash-table==5.0.0
+dm_control==1.0.23
+einops==0.4.1
+ptyprocess==0.7.0
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+nbformat==5.10.4
+decorator==4.4.2
+importlib_metadata==8.5.0
+numcodecs==0.12.1
+requests==2.32.3
+ipywidgets==8.1.5
+urllib3==2.2.3
+platformdirs==4.3.6
+imageio-ffmpeg==0.5.1
+dash==2.18.2
+Cython==0.29.35
+mujoco-py==2.1.2.14
+mpmath==1.3.0
+typing_extensions==4.12.2
+dm-tree==0.1.8
+imageio==2.35.1
+dill==0.3.5.1
+joblib==1.4.2
+attrs==24.3.0
+torchvision==0.19.1+cu121
+kiwisolver==1.4.7
+numpy==1.23.5
+pillow==10.2.0
+nvidia-cuda-cupti-cu12==12.1.105
+pip==24.2
+llvmlite==0.39.1
+tenacity==9.0.0
+scipy==1.10.1
+charset-normalizer==3.4.0
+matplotlib==3.7.5
+jupyterlab_widgets==3.0.13
+patchelf==0.17.2.0
+wcwidth==0.2.13
+pexpect==4.9.0
+plotly==5.24.1
+ipdb==0.13.13
+referencing==0.35.1
+mujoco==3.2.3
+portalocker==3.0.0
+jedi==0.19.2
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+nvidia-nvtx-cu12==12.1.105
+python-dateutil==2.9.0.post0
+packaging==24.2
+click==8.1.7
+pycparser==2.22
+blinker==1.8.2
+sympy==1.13.1
+Werkzeug==3.0.6
+six==1.17.0
+stack-data==0.6.3
+gpustat==1.1.1
+threadpoolctl==3.5.0
+traitlets==5.14.3
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.1.105
+zarr==2.12.0
+pandas==2.0.3
+wheel==0.44.0
+nvidia-nccl-cu12==2.20.5
+comm==0.2.2
+proglog==0.1.10
+parso==0.8.4
+open3d==0.18.0
+nvidia-cusparse-cu12==12.1.0.106
+antlr4-python3-runtime==4.9.3
+fvcore==0.1.5.post20221221
+natsort==8.4.0
+cloudpickle==3.1.0
+ConfigArgParse==1.7
+fonttools==4.55.3
+etils==1.3.0
+torchaudio==2.4.1+cu121
+pickleshare==0.7.5
+eval_type_backport==0.2.0
+executing==2.1.0
+addict==2.4.0
+protobuf==5.29.1
+setproctitle==1.3.4
+Flask==3.0.3
+filelock==3.13.1
+regex==2024.11.6
+PyOpenGL==3.1.7
+nvidia-curand-cu12==10.3.2.106
+matplotlib-inline==0.1.7
+transforms3d==0.4.2
+pydantic==2.10.3
+cffi==1.17.1
+rpds-py==0.20.1
+diffusers==0.11.1
+docker-pycreds==0.4.0
+pyparsing==3.1.4
+absl-py==2.1.0
+nvidia-cufft-cu12==11.0.2.54
+contourpy==1.1.1
+fastjsonschema==2.21.1
+pkgutil_resolve_name==1.3.10
+dash-html-components==2.0.0
+asciitree==0.3.3
+fasteners==0.19
+glfw==2.8.0
+omegaconf==2.3.0
+asttokens==3.0.0
+moviepy==1.0.3
+GitPython==3.1.43
+opencv-python==4.10.0.84
+zipp==3.20.2
+tabulate==0.9.0
+iopath==0.1.10
+labmaze==1.0.6
+numba==0.56.4
+nvidia-cuda-runtime-cu12==12.1.105
+dm-env==1.6
+cycler==0.12.1
+dash-core-components==2.0.0
+networkx==3.0
+Pygments==2.18.0
+retrying==1.3.4
+jsonschema-specifications==2023.12.1
+setuptools==59.5.0
+av==12.3.0
+itsdangerous==2.2.0
+wandb==0.19.1
+nest-asyncio==1.6.0
+triton==3.0.0
+nvidia-ml-py==12.560.30
+nvidia-cuda-nvrtc-cu12==12.1.105
+tomli==2.2.1
+tqdm==4.67.1
+termcolor==2.4.0
+hydra-core==1.2.0
+smmap==5.0.1
+ipython==8.12.3
+pyquaternion==0.9.9
+idna==3.10
+yacs==0.1.8
+tzdata==2024.2
+torch==2.4.1+cu121
+widgetsnbextension==4.0.13
+scikit-learn==1.3.2
+sapien==2.2.1
+nvidia-cublas-cu12==12.1.3.1
+huggingface-hub==0.21.0
+gitdb==4.0.11
+importlib_resources==6.4.5
+pydantic_core==2.27.1
+jupyter_core==5.7.2
+sentry-sdk==2.19.2
+nvidia-cudnn-cu12==9.1.0.70
+fsspec==2024.2.0
+psutil==6.1.0
+pytz==2024.2
+backcall==0.2.0
+nvidia-cusolver-cu12==11.4.5.107
+lxml==5.3.0
+blessed==1.20.0
+pure_eval==0.2.3
+diffusion_policy_3d==0.0.0
+dexart==0.1.0
+gym==0.21.0
+metaworld==0.1.0
+mj-envs==1.0.0
+mjrl==1.0.0
+pytorch3d==0.7.6
+diffusion_policy_3d==0.0.0

adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-19 02:23:11,035 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Configure stats pid to 4160108
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-internal.log
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:init():644] calling init triggers
+2024-12-19 02:23:11,036 INFO    MainThread:4160108 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'adroit_pen', 'task_name': 'pen', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'pen', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_pen_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_pen', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'exp_name': 'adroit_pen-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_pen-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen', 'wandb_name_base': '2024.12.19-02.23.08_train_dp3_adroit_pen'}}
+2024-12-19 02:23:11,037 INFO    MainThread:4160108 [wandb_init.py:init():680] starting backend
+2024-12-19 02:23:11,037 INFO    MainThread:4160108 [wandb_init.py:init():684] sending inform_init request
+2024-12-19 02:23:11,063 INFO    MainThread:4160108 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-19 02:23:11,064 INFO    MainThread:4160108 [wandb_init.py:init():697] backend started and connected
+2024-12-19 02:23:11,069 INFO    MainThread:4160108 [wandb_init.py:init():790] updated telemetry
+2024-12-19 02:23:11,259 INFO    MainThread:4160108 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-19 02:23:13,842 INFO    MainThread:4160108 [wandb_init.py:init():874] starting run threads in backend
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-19 02:23:13,966 INFO    MainThread:4160108 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-19 02:23:13,969 INFO    MainThread:4160108 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-19 02:23:13,969 INFO    MainThread:4160108 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0'}
+2024-12-19 03:39:20,847 WARNING MsgRouterThr:4160108 [router.py:message_loop():75] message_loop has been closed

dexart_bucket-dp3-1219_seed0/.hydra/overrides.yaml

@@ -0,0 +1,7 @@
+- task=dexart_bucket
+- training.debug=False
+- training.seed=0
+- training.device=cuda:0
+- exp_name=dexart_bucket-dp3-1219
+- logging.mode=online
+- checkpoint.save_ckpt=True

dexart_bucket-dp3-1219_seed0/train.log

@@ -0,0 +1 @@
+[2024-12-19 16:44:56,429][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.551390e+08

metaworld_assembly-dp3-1218_seed0/wandb/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2024-12-18T14:28:52.397338454+08:00","level":"INFO","msg":"using version","core version":"0.19.1"}
+{"time":"2024-12-18T14:28:52.397351034+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug-core.log"}
+{"time":"2024-12-18T14:28:52.609650827+08:00","level":"INFO","msg":"created new stream","id":"63943iru"}
+{"time":"2024-12-18T14:28:52.609680037+08:00","level":"INFO","msg":"stream: started","id":"63943iru"}
+{"time":"2024-12-18T14:28:52.609755478+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"63943iru"}
+{"time":"2024-12-18T14:28:52.609795528+08:00","level":"INFO","msg":"handler: started","stream_id":"63943iru"}
+{"time":"2024-12-18T14:28:52.611313401+08:00","level":"INFO","msg":"sender: started","stream_id":"63943iru"}
+{"time":"2024-12-18T14:28:55.113722346+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-12-18T16:42:47.860321056+08:00","level":"INFO","msg":"stream: closing","id":"63943iru"}
+{"time":"2024-12-18T16:42:47.860366545+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-12-18T16:42:47.861825165+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-12-18T16:42:51.203826373+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-12-18T16:42:54.393251212+08:00","level":"INFO","msg":"handler: closed","stream_id":"63943iru"}
+{"time":"2024-12-18T16:42:54.393317331+08:00","level":"INFO","msg":"sender: closed","stream_id":"63943iru"}
+{"time":"2024-12-18T16:42:54.393312931+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"63943iru"}
+{"time":"2024-12-18T16:42:54.393483768+08:00","level":"INFO","msg":"stream: closed","id":"63943iru"}

metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/config.yaml

@@ -0,0 +1,242 @@
+_wandb:
+    value:
+        cli_version: 0.19.1
+        m: []
+        python_version: 3.8.20
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.8.20
+            "5": 0.19.1
+            "8":
+                - 5
+            "12": 0.19.1
+            "13": linux-x86_64
+checkpoint:
+    value:
+        save_ckpt: true
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+            k: 1
+            mode: max
+            monitor_key: test_mean_score
+dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: true
+dataset_obs_steps:
+    value: 2
+ema:
+    value:
+        _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+        inv_gamma: 1
+        max_value: 0.9999
+        min_value: 0
+        power: 0.75
+        update_after_step: 0
+exp_name:
+    value: metaworld_assembly-dp3-1218
+horizon:
+    value: 4
+keypoint_visible_rate:
+    value: 1
+logging:
+    value:
+        group: metaworld_assembly-dp3-1218
+        id: null
+        mode: online
+        name: "0"
+        project: dp3
+        resume: true
+        tags:
+            - dp3
+multi_run:
+    value:
+        run_dir: data/outputs/2024.12.18/14.28.49_train_dp3_assembly
+        wandb_name_base: 2024.12.18-14.28.49_train_dp3_assembly
+n_action_steps:
+    value: 3
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 2
+name:
+    value: train_dp3
+obs_as_global_cond:
+    value: true
+optimizer:
+    value:
+        _target_: torch.optim.AdamW
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0
+policy:
+    value:
+        _target_: diffusion_policy_3d.policy.dp3.DP3
+        condition_type: film
+        crop_shape:
+            - 80
+            - 80
+        diffusion_step_embed_dim: 128
+        down_dims:
+            - 512
+            - 1024
+            - 2048
+        encoder_output_dim: 64
+        horizon: 4
+        kernel_size: 5
+        n_action_steps: 3
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler:
+            _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+            beta_end: 0.02
+            beta_schedule: squaredcos_cap_v2
+            beta_start: 0.0001
+            clip_sample: true
+            num_train_timesteps: 100
+            prediction_type: sample
+            set_alpha_to_one: true
+            steps_offset: 0
+        num_inference_steps: 10
+        obs_as_global_cond: true
+        pointcloud_encoder_cfg:
+            final_norm: layernorm
+            in_channels: 3
+            normal_channel: false
+            out_channels: 64
+            use_layernorm: true
+        pointnet_type: pointnet
+        shape_meta:
+            action:
+                shape:
+                    - 4
+            obs:
+                agent_pos:
+                    shape:
+                        - 9
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        use_down_condition: true
+        use_mid_condition: true
+        use_pc_color: false
+        use_point_crop: true
+        use_up_condition: true
+shape_meta:
+    value:
+        action:
+            shape:
+                - 4
+        obs:
+            agent_pos:
+                shape:
+                    - 9
+                type: low_dim
+            point_cloud:
+                shape:
+                    - 512
+                    - 3
+                type: point_cloud
+task:
+    value:
+        dataset:
+            _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset
+            horizon: 4
+            max_train_episodes: 90
+            pad_after: 2
+            pad_before: 1
+            seed: 42
+            val_ratio: 0.02
+            zarr_path: /data/haojun/datasets/3d-dp/metaworld_assembly_expert.zarr
+        env_runner:
+            _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner
+            device: cuda:0
+            eval_episodes: 20
+            fps: 10
+            n_action_steps: 3
+            n_envs: null
+            n_obs_steps: 2
+            n_test: null
+            n_train: null
+            task_name: assembly
+            use_point_crop: true
+        name: assembly
+        shape_meta:
+            action:
+                shape:
+                    - 4
+            obs:
+                agent_pos:
+                    shape:
+                        - 9
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        task_name: train_dp3
+task_name:
+    value: assembly
+training:
+    value:
+        checkpoint_every: 200
+        debug: false
+        device: cuda:0
+        gradient_accumulate_every: 1
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 3000
+        resume: true
+        rollout_every: 200
+        sample_every: 5
+        seed: 0
+        tqdm_interval_sec: 1
+        use_ema: true
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: false

metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/wandb-summary.json

@@ -0,0 +1 @@
+{"lr":0,"epoch":2999,"SR_test_L5":1,"_wandb":{"runtime":8035},"mean_traj_rewards":30884.26171460612,"mean_success_rates":1,"_runtime":8035.464377365,"global_step":44999,"bc_loss":2.0856390392509638e-07,"_step":44999,"train_action_mse_error":4.6260427666311443e-07,"SR_test_L3":1,"_timestamp":1.734511367852143e+09,"train_loss":2.420270874381458e-07,"test_mean_score":1}

metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_setup.py:_flush():68] Configure stats pid to 2428026
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug-internal.log
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:init():644] calling init triggers
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'assembly', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'assembly', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_assembly_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'assembly', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_assembly-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_assembly-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/14.28.49_train_dp3_assembly', 'wandb_name_base': '2024.12.18-14.28.49_train_dp3_assembly'}}
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:init():680] starting backend
+2024-12-18 14:28:52,369 INFO    MainThread:2428026 [wandb_init.py:init():684] sending inform_init request
+2024-12-18 14:28:52,395 INFO    MainThread:2428026 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-18 14:28:52,395 INFO    MainThread:2428026 [wandb_init.py:init():697] backend started and connected
+2024-12-18 14:28:52,400 INFO    MainThread:2428026 [wandb_init.py:init():790] updated telemetry
+2024-12-18 14:28:52,577 INFO    MainThread:2428026 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-18 14:28:55,105 INFO    MainThread:2428026 [wandb_init.py:init():874] starting run threads in backend
+2024-12-18 14:28:55,225 INFO    MainThread:2428026 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-18 14:28:55,225 INFO    MainThread:2428026 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-18 14:28:55,226 INFO    MainThread:2428026 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-18 14:28:55,226 INFO    MainThread:2428026 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-18 14:28:55,228 INFO    MainThread:2428026 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-18 14:28:55,228 INFO    MainThread:2428026 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0'}
+2024-12-18 16:42:47,860 WARNING MsgRouterThr:2428026 [router.py:message_loop():75] message_loop has been closed

metaworld_bin-picking-dp3-1218_seed0/train.log

@@ -0,0 +1 @@
+[2024-12-18 20:53:06,678][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08

metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/files/requirements.txt

@@ -0,0 +1,175 @@
+diffusion_policy_3d==0.0.0
+certifi==2024.12.14
+PyYAML==6.0.2
+dash-table==5.0.0
+dm_control==1.0.23
+einops==0.4.1
+ptyprocess==0.7.0
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+nbformat==5.10.4
+decorator==4.4.2
+importlib_metadata==8.5.0
+numcodecs==0.12.1
+requests==2.32.3
+ipywidgets==8.1.5
+urllib3==2.2.3
+platformdirs==4.3.6
+imageio-ffmpeg==0.5.1
+dash==2.18.2
+Cython==0.29.35
+mujoco-py==2.1.2.14
+mpmath==1.3.0
+typing_extensions==4.12.2
+dm-tree==0.1.8
+imageio==2.35.1
+dill==0.3.5.1
+joblib==1.4.2
+attrs==24.3.0
+torchvision==0.19.1+cu121
+kiwisolver==1.4.7
+numpy==1.23.5
+pillow==10.2.0
+nvidia-cuda-cupti-cu12==12.1.105
+pip==24.2
+llvmlite==0.39.1
+tenacity==9.0.0
+scipy==1.10.1
+charset-normalizer==3.4.0
+matplotlib==3.7.5
+jupyterlab_widgets==3.0.13
+patchelf==0.17.2.0
+wcwidth==0.2.13
+pexpect==4.9.0
+plotly==5.24.1
+ipdb==0.13.13
+referencing==0.35.1
+mujoco==3.2.3
+portalocker==3.0.0
+jedi==0.19.2
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+nvidia-nvtx-cu12==12.1.105
+python-dateutil==2.9.0.post0
+packaging==24.2
+click==8.1.7
+pycparser==2.22
+blinker==1.8.2
+sympy==1.13.1
+Werkzeug==3.0.6
+six==1.17.0
+stack-data==0.6.3
+gpustat==1.1.1
+threadpoolctl==3.5.0
+traitlets==5.14.3
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.1.105
+zarr==2.12.0
+pandas==2.0.3
+wheel==0.44.0
+nvidia-nccl-cu12==2.20.5
+comm==0.2.2
+proglog==0.1.10
+parso==0.8.4
+open3d==0.18.0
+nvidia-cusparse-cu12==12.1.0.106
+antlr4-python3-runtime==4.9.3
+fvcore==0.1.5.post20221221
+natsort==8.4.0
+cloudpickle==3.1.0
+ConfigArgParse==1.7
+fonttools==4.55.3
+etils==1.3.0
+torchaudio==2.4.1+cu121
+pickleshare==0.7.5
+eval_type_backport==0.2.0
+executing==2.1.0
+addict==2.4.0
+protobuf==5.29.1
+setproctitle==1.3.4
+Flask==3.0.3
+filelock==3.13.1
+regex==2024.11.6
+PyOpenGL==3.1.7
+nvidia-curand-cu12==10.3.2.106
+matplotlib-inline==0.1.7
+transforms3d==0.4.2
+pydantic==2.10.3
+cffi==1.17.1
+rpds-py==0.20.1
+diffusers==0.11.1
+docker-pycreds==0.4.0
+pyparsing==3.1.4
+absl-py==2.1.0
+nvidia-cufft-cu12==11.0.2.54
+contourpy==1.1.1
+fastjsonschema==2.21.1
+pkgutil_resolve_name==1.3.10
+dash-html-components==2.0.0
+asciitree==0.3.3
+fasteners==0.19
+glfw==2.8.0
+omegaconf==2.3.0
+asttokens==3.0.0
+moviepy==1.0.3
+GitPython==3.1.43
+opencv-python==4.10.0.84
+zipp==3.20.2
+tabulate==0.9.0
+iopath==0.1.10
+labmaze==1.0.6
+numba==0.56.4
+nvidia-cuda-runtime-cu12==12.1.105
+dm-env==1.6
+cycler==0.12.1
+dash-core-components==2.0.0
+networkx==3.0
+Pygments==2.18.0
+retrying==1.3.4
+jsonschema-specifications==2023.12.1
+setuptools==59.5.0
+av==12.3.0
+itsdangerous==2.2.0
+wandb==0.19.1
+nest-asyncio==1.6.0
+triton==3.0.0
+nvidia-ml-py==12.560.30
+nvidia-cuda-nvrtc-cu12==12.1.105
+tomli==2.2.1
+tqdm==4.67.1
+termcolor==2.4.0
+hydra-core==1.2.0
+smmap==5.0.1
+ipython==8.12.3
+pyquaternion==0.9.9
+idna==3.10
+yacs==0.1.8
+tzdata==2024.2
+torch==2.4.1+cu121
+widgetsnbextension==4.0.13
+scikit-learn==1.3.2
+sapien==2.2.1
+nvidia-cublas-cu12==12.1.3.1
+huggingface-hub==0.21.0
+gitdb==4.0.11
+importlib_resources==6.4.5
+pydantic_core==2.27.1
+jupyter_core==5.7.2
+sentry-sdk==2.19.2
+nvidia-cudnn-cu12==9.1.0.70
+fsspec==2024.2.0
+psutil==6.1.0
+pytz==2024.2
+backcall==0.2.0
+nvidia-cusolver-cu12==11.4.5.107
+lxml==5.3.0
+blessed==1.20.0
+pure_eval==0.2.3
+diffusion_policy_3d==0.0.0
+dexart==0.1.0
+gym==0.21.0
+metaworld==0.1.0
+mj-envs==1.0.0
+mjrl==1.0.0
+pytorch3d==0.7.6
+diffusion_policy_3d==0.0.0

metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_setup.py:_flush():68] Configure stats pid to 2962987
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug-internal.log
+2024-12-18 20:53:14,968 INFO    MainThread:2962987 [wandb_init.py:init():644] calling init triggers
+2024-12-18 20:53:14,969 INFO    MainThread:2962987 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'bin-picking', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'bin-picking', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_bin-picking_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'bin-picking', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_bin-picking-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_bin-picking-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/20.53.12_train_dp3_bin-picking', 'wandb_name_base': '2024.12.18-20.53.12_train_dp3_bin-picking'}}
+2024-12-18 20:53:14,969 INFO    MainThread:2962987 [wandb_init.py:init():680] starting backend
+2024-12-18 20:53:14,969 INFO    MainThread:2962987 [wandb_init.py:init():684] sending inform_init request
+2024-12-18 20:53:14,996 INFO    MainThread:2962987 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-18 20:53:14,997 INFO    MainThread:2962987 [wandb_init.py:init():697] backend started and connected
+2024-12-18 20:53:15,002 INFO    MainThread:2962987 [wandb_init.py:init():790] updated telemetry
+2024-12-18 20:53:15,163 INFO    MainThread:2962987 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-18 20:53:17,822 INFO    MainThread:2962987 [wandb_init.py:init():874] starting run threads in backend
+2024-12-18 20:53:17,936 INFO    MainThread:2962987 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-18 20:53:17,937 INFO    MainThread:2962987 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-18 20:53:17,937 INFO    MainThread:2962987 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-18 20:53:17,937 INFO    MainThread:2962987 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-18 20:53:17,939 INFO    MainThread:2962987 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-18 20:53:17,939 INFO    MainThread:2962987 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0'}
+2024-12-19 00:26:54,104 WARNING MsgRouterThr:2962987 [router.py:message_loop():75] message_loop has been closed

metaworld_box-close-dp3-1218_seed0/.hydra/hydra.yaml

@@ -0,0 +1,163 @@
+hydra:
+  run:
+    dir: data/outputs/metaworld_box-close-dp3-1218_seed0
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.run.dir=data/outputs/metaworld_box-close-dp3-1218_seed0
+    - hydra.mode=RUN
+    task:
+    - task=metaworld_box-close
+    - training.debug=False
+    - training.seed=0
+    - training.device=cuda:0
+    - exp_name=metaworld_box-close-dp3-1218
+    - logging.mode=online
+    - checkpoint.save_ckpt=True
+  job:
+    name: train
+    chdir: null
+    override_dirname: checkpoint.save_ckpt=True,exp_name=metaworld_box-close-dp3-1218,logging.mode=online,task=metaworld_box-close,training.debug=False,training.device=cuda:0,training.seed=0
+    id: ???
+    num: ???
+    config_name: dp3.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_box-close-dp3-1218_seed0
+    choices:
+      task: metaworld_box-close
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/output.log

@@ -0,0 +1,46 @@
+                                                                                                                             
+test_mean_score: 0.0
+mean_durations: 4.828114485740661
+std_durations: 0.01738945887153302
+test_mean_score: 0.65
+mean_durations: 4.873560278303922
+std_durations: 0.019777209321623128
+test_mean_score: 0.0
+mean_durations: 4.854758024774492
+std_durations: 0.007352221929266012
+test_mean_score: 0.5
+mean_durations: 4.845281571056693
+std_durations: 0.008404259880539429
+test_mean_score: 0.6
+mean_durations: 4.851413570158184
+std_durations: 0.005525312932477265
+test_mean_score: 0.45
+mean_durations: 4.8451534550637
+std_durations: 0.005656914477641985
+test_mean_score: 0.55
+mean_durations: 4.850002092681825
+std_durations: 0.003998623920949613
+test_mean_score: 0.65
+mean_durations: 4.8544698669575155
+std_durations: 0.004563551825275274
+test_mean_score: 0.3
+mean_durations: 4.8519475953653455
+std_durations: 0.005867139433633655
+test_mean_score: 0.55
+mean_durations: 4.849922423437238
+std_durations: 0.024024316288814206
+test_mean_score: 0.45
+mean_durations: 4.851007116865366
+std_durations: 0.009671100741088231
+test_mean_score: 0.5
+mean_durations: 4.842636080086232
+std_durations: 0.0025684137610381937
+test_mean_score: 0.35
+mean_durations: 4.842003949917853
+std_durations: 0.004577108168226907
+test_mean_score: 0.2
+mean_durations: 4.850726788770407
+std_durations: 0.0029142146415510766
+test_mean_score: 0.45
+mean_durations: 4.84709279127419
+std_durations: 0.003142567013560075

metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/wandb-summary.json

@@ -0,0 +1 @@
+{"mean_success_rates":0.45,"SR_test_L5":0.6,"lr":0,"_step":44999,"bc_loss":1.1081001503043808e-06,"epoch":2999,"SR_test_L3":0.6333333333333333,"mean_durations":4.84709279127419,"global_step":44999,"train_action_mse_error":1.8008979168371297e-05,"std_durations":0.003142567013560075,"_timestamp":1.7345537128343365e+09,"train_loss":1.2438786446485514e-06,"test_mean_score":0.45,"_runtime":8126.141499735,"mean_traj_rewards":13042.742712645655,"_wandb":{"runtime":8126}}

metaworld_button-press-topdown-dp3-0322_seed0/checkpoints/info.txt

@@ -0,0 +1 @@
+epoch=0200-test_mean_score=1.000.ckpt

metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/config.yaml

@@ -0,0 +1,242 @@
+_wandb:
+    value:
+        cli_version: 0.19.1
+        m: []
+        python_version: 3.8.20
+        t:
+            "1":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "2":
+                - 1
+                - 5
+                - 41
+                - 49
+                - 50
+                - 53
+                - 55
+                - 83
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.8.20
+            "5": 0.19.1
+            "8":
+                - 5
+            "12": 0.19.1
+            "13": linux-x86_64
+checkpoint:
+    value:
+        save_ckpt: true
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+            k: 1
+            mode: max
+            monitor_key: test_mean_score
+dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: true
+dataset_obs_steps:
+    value: 2
+ema:
+    value:
+        _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+        inv_gamma: 1
+        max_value: 0.9999
+        min_value: 0
+        power: 0.75
+        update_after_step: 0
+exp_name:
+    value: metaworld_button-press-topdown-dp3-0322
+horizon:
+    value: 4
+keypoint_visible_rate:
+    value: 1
+logging:
+    value:
+        group: metaworld_button-press-topdown-dp3-0322
+        id: null
+        mode: online
+        name: "0"
+        project: dp3
+        resume: true
+        tags:
+            - dp3
+multi_run:
+    value:
+        run_dir: data/outputs/2024.12.18/02.29.37_train_dp3_button-press-topdown
+        wandb_name_base: 2024.12.18-02.29.37_train_dp3_button-press-topdown
+n_action_steps:
+    value: 3
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 2
+name:
+    value: train_dp3
+obs_as_global_cond:
+    value: true
+optimizer:
+    value:
+        _target_: torch.optim.AdamW
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0
+policy:
+    value:
+        _target_: diffusion_policy_3d.policy.dp3.DP3
+        condition_type: film
+        crop_shape:
+            - 80
+            - 80
+        diffusion_step_embed_dim: 128
+        down_dims:
+            - 512
+            - 1024
+            - 2048
+        encoder_output_dim: 64
+        horizon: 4
+        kernel_size: 5
+        n_action_steps: 3
+        n_groups: 8
+        n_obs_steps: 2
+        noise_scheduler:
+            _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+            beta_end: 0.02
+            beta_schedule: squaredcos_cap_v2
+            beta_start: 0.0001
+            clip_sample: true
+            num_train_timesteps: 100
+            prediction_type: sample
+            set_alpha_to_one: true
+            steps_offset: 0
+        num_inference_steps: 10
+        obs_as_global_cond: true
+        pointcloud_encoder_cfg:
+            final_norm: layernorm
+            in_channels: 3
+            normal_channel: false
+            out_channels: 64
+            use_layernorm: true
+        pointnet_type: pointnet
+        shape_meta:
+            action:
+                shape:
+                    - 4
+            obs:
+                agent_pos:
+                    shape:
+                        - 9
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        use_down_condition: true
+        use_mid_condition: true
+        use_pc_color: false
+        use_point_crop: true
+        use_up_condition: true
+shape_meta:
+    value:
+        action:
+            shape:
+                - 4
+        obs:
+            agent_pos:
+                shape:
+                    - 9
+                type: low_dim
+            point_cloud:
+                shape:
+                    - 512
+                    - 3
+                type: point_cloud
+task:
+    value:
+        dataset:
+            _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset
+            horizon: 4
+            max_train_episodes: 90
+            pad_after: 2
+            pad_before: 1
+            seed: 42
+            val_ratio: 0.02
+            zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-topdown_expert.zarr
+        env_runner:
+            _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner
+            device: cuda:0
+            eval_episodes: 20
+            fps: 10
+            n_action_steps: 3
+            n_envs: null
+            n_obs_steps: 2
+            n_test: null
+            n_train: null
+            task_name: button-press-topdown
+            use_point_crop: true
+        name: button-press-topdown
+        shape_meta:
+            action:
+                shape:
+                    - 4
+            obs:
+                agent_pos:
+                    shape:
+                        - 9
+                    type: low_dim
+                point_cloud:
+                    shape:
+                        - 512
+                        - 3
+                    type: point_cloud
+        task_name: train_dp3
+task_name:
+    value: button-press-topdown
+training:
+    value:
+        checkpoint_every: 200
+        debug: false
+        device: cuda:0
+        gradient_accumulate_every: 1
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 3000
+        resume: true
+        rollout_every: 200
+        sample_every: 5
+        seed: 0
+        tqdm_interval_sec: 1
+        use_ema: true
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 128
+        num_workers: 8
+        persistent_workers: false
+        pin_memory: true
+        shuffle: false

metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/requirements.txt

@@ -0,0 +1,175 @@
+diffusion_policy_3d==0.0.0
+certifi==2024.12.14
+PyYAML==6.0.2
+dash-table==5.0.0
+dm_control==1.0.23
+einops==0.4.1
+ptyprocess==0.7.0
+jsonschema==4.23.0
+prompt_toolkit==3.0.48
+nbformat==5.10.4
+decorator==4.4.2
+importlib_metadata==8.5.0
+numcodecs==0.12.1
+requests==2.32.3
+ipywidgets==8.1.5
+urllib3==2.2.3
+platformdirs==4.3.6
+imageio-ffmpeg==0.5.1
+dash==2.18.2
+Cython==0.29.35
+mujoco-py==2.1.2.14
+mpmath==1.3.0
+typing_extensions==4.12.2
+dm-tree==0.1.8
+imageio==2.35.1
+dill==0.3.5.1
+joblib==1.4.2
+attrs==24.3.0
+torchvision==0.19.1+cu121
+kiwisolver==1.4.7
+numpy==1.23.5
+pillow==10.2.0
+nvidia-cuda-cupti-cu12==12.1.105
+pip==24.2
+llvmlite==0.39.1
+tenacity==9.0.0
+scipy==1.10.1
+charset-normalizer==3.4.0
+matplotlib==3.7.5
+jupyterlab_widgets==3.0.13
+patchelf==0.17.2.0
+wcwidth==0.2.13
+pexpect==4.9.0
+plotly==5.24.1
+ipdb==0.13.13
+referencing==0.35.1
+mujoco==3.2.3
+portalocker==3.0.0
+jedi==0.19.2
+Jinja2==3.1.3
+MarkupSafe==2.1.5
+nvidia-nvtx-cu12==12.1.105
+python-dateutil==2.9.0.post0
+packaging==24.2
+click==8.1.7
+pycparser==2.22
+blinker==1.8.2
+sympy==1.13.1
+Werkzeug==3.0.6
+six==1.17.0
+stack-data==0.6.3
+gpustat==1.1.1
+threadpoolctl==3.5.0
+traitlets==5.14.3
+annotated-types==0.7.0
+nvidia-nvjitlink-cu12==12.1.105
+zarr==2.12.0
+pandas==2.0.3
+wheel==0.44.0
+nvidia-nccl-cu12==2.20.5
+comm==0.2.2
+proglog==0.1.10
+parso==0.8.4
+open3d==0.18.0
+nvidia-cusparse-cu12==12.1.0.106
+antlr4-python3-runtime==4.9.3
+fvcore==0.1.5.post20221221
+natsort==8.4.0
+cloudpickle==3.1.0
+ConfigArgParse==1.7
+fonttools==4.55.3
+etils==1.3.0
+torchaudio==2.4.1+cu121
+pickleshare==0.7.5
+eval_type_backport==0.2.0
+executing==2.1.0
+addict==2.4.0
+protobuf==5.29.1
+setproctitle==1.3.4
+Flask==3.0.3
+filelock==3.13.1
+regex==2024.11.6
+PyOpenGL==3.1.7
+nvidia-curand-cu12==10.3.2.106
+matplotlib-inline==0.1.7
+transforms3d==0.4.2
+pydantic==2.10.3
+cffi==1.17.1
+rpds-py==0.20.1
+diffusers==0.11.1
+docker-pycreds==0.4.0
+pyparsing==3.1.4
+absl-py==2.1.0
+nvidia-cufft-cu12==11.0.2.54
+contourpy==1.1.1
+fastjsonschema==2.21.1
+pkgutil_resolve_name==1.3.10
+dash-html-components==2.0.0
+asciitree==0.3.3
+fasteners==0.19
+glfw==2.8.0
+omegaconf==2.3.0
+asttokens==3.0.0
+moviepy==1.0.3
+GitPython==3.1.43
+opencv-python==4.10.0.84
+zipp==3.20.2
+tabulate==0.9.0
+iopath==0.1.10
+labmaze==1.0.6
+numba==0.56.4
+nvidia-cuda-runtime-cu12==12.1.105
+dm-env==1.6
+cycler==0.12.1
+dash-core-components==2.0.0
+networkx==3.0
+Pygments==2.18.0
+retrying==1.3.4
+jsonschema-specifications==2023.12.1
+setuptools==59.5.0
+av==12.3.0
+itsdangerous==2.2.0
+wandb==0.19.1
+nest-asyncio==1.6.0
+triton==3.0.0
+nvidia-ml-py==12.560.30
+nvidia-cuda-nvrtc-cu12==12.1.105
+tomli==2.2.1
+tqdm==4.67.1
+termcolor==2.4.0
+hydra-core==1.2.0
+smmap==5.0.1
+ipython==8.12.3
+pyquaternion==0.9.9
+idna==3.10
+yacs==0.1.8
+tzdata==2024.2
+torch==2.4.1+cu121
+widgetsnbextension==4.0.13
+scikit-learn==1.3.2
+sapien==2.2.1
+nvidia-cublas-cu12==12.1.3.1
+huggingface-hub==0.21.0
+gitdb==4.0.11
+importlib_resources==6.4.5
+pydantic_core==2.27.1
+jupyter_core==5.7.2
+sentry-sdk==2.19.2
+nvidia-cudnn-cu12==9.1.0.70
+fsspec==2024.2.0
+psutil==6.1.0
+pytz==2024.2
+backcall==0.2.0
+nvidia-cusolver-cu12==11.4.5.107
+lxml==5.3.0
+blessed==1.20.0
+pure_eval==0.2.3
+diffusion_policy_3d==0.0.0
+dexart==0.1.0
+gym==0.21.0
+metaworld==0.1.0
+mj-envs==1.0.0
+mjrl==1.0.0
+pytorch3d==0.7.6
+diffusion_policy_3d==0.0.0

metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os":  "Linux-5.15.0-117-generic-x86_64-with-glibc2.17",
+  "python":  "CPython 3.8.20",
+  "startedAt":  "2024-12-17T18:29:40.205080Z",
+  "args":  [
+    "--config-name=dp3.yaml",
+    "task=metaworld_button-press-topdown",
+    "hydra.run.dir=data/outputs/metaworld_button-press-topdown-dp3-0322_seed0",
+    "training.debug=False",
+    "training.seed=0",
+    "training.device=cuda:0",
+    "exp_name=metaworld_button-press-topdown-dp3-0322",
+    "logging.mode=online",
+    "checkpoint.save_ckpt=True"
+  ],
+  "program":  "train.py",
+  "codePath":  "3D-Diffusion-Policy/train.py",
+  "git":  {
+    "remote":  "git@github.com:YanjieZe/3D-Diffusion-Policy.git",
+    "commit":  "3153fb32c17308fc828015156480b77f6b2e666a"
+  },
+  "email":  "chenhaojun001@gmail.com",
+  "root":  "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0",
+  "host":  "Server-3090",
+  "executable":  "/home/chenhaojun/miniconda3/envs/dp3/bin/python",
+  "codePathLocal":  "train.py",
+  "cpu_count":  128,
+  "cpu_count_logical":  256,
+  "gpu":  "NVIDIA GeForce RTX 3090",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "943412031488",
+      "used":  "868376309760"
+    }
+  },
+  "memory":  {
+    "total":  "1081980669952"
+  },
+  "cpu":  {
+    "count":  128,
+    "countLogical":  256
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA GeForce RTX 3090",
+      "memoryTotal":  "25769803776",
+      "cudaCores":  10496,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "12.5"
+}

metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-summary.json

@@ -0,0 +1 @@
+{"global_step":44999,"SR_test_L5":1,"_runtime":7859.566856332,"train_loss":5.629131862860959e-07,"test_mean_score":1,"mean_traj_rewards":29227.664171579825,"lr":0,"train_action_mse_error":1.17400359158637e-05,"epoch":2999,"_step":44999,"SR_test_L3":1,"bc_loss":6.396743970071839e-07,"_wandb":{"runtime":7859},"_timestamp":1.7344680397637768e+09,"mean_success_rates":1}

metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_setup.py:_flush():68] Configure stats pid to 2040078
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug-internal.log
+2024-12-18 02:29:40,171 INFO    MainThread:2040078 [wandb_init.py:init():644] calling init triggers
+2024-12-18 02:29:40,172 INFO    MainThread:2040078 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'button-press-topdown', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'button-press-topdown', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_button-press-topdown_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'button-press-topdown', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_button-press-topdown-dp3-0322', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_button-press-topdown-dp3-0322', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/02.29.37_train_dp3_button-press-topdown', 'wandb_name_base': '2024.12.18-02.29.37_train_dp3_button-press-topdown'}}
+2024-12-18 02:29:40,172 INFO    MainThread:2040078 [wandb_init.py:init():680] starting backend
+2024-12-18 02:29:40,172 INFO    MainThread:2040078 [wandb_init.py:init():684] sending inform_init request
+2024-12-18 02:29:40,204 INFO    MainThread:2040078 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-18 02:29:40,204 INFO    MainThread:2040078 [wandb_init.py:init():697] backend started and connected
+2024-12-18 02:29:40,209 INFO    MainThread:2040078 [wandb_init.py:init():790] updated telemetry
+2024-12-18 02:29:40,368 INFO    MainThread:2040078 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-18 02:29:41,789 INFO    MainThread:2040078 [wandb_init.py:init():874] starting run threads in backend
+2024-12-18 02:29:41,910 INFO    MainThread:2040078 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-18 02:29:41,910 INFO    MainThread:2040078 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-18 02:29:41,911 INFO    MainThread:2040078 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-18 02:29:41,911 INFO    MainThread:2040078 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-18 02:29:41,913 INFO    MainThread:2040078 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-18 02:29:41,913 INFO    MainThread:2040078 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0'}
+2024-12-18 04:40:39,772 WARNING MsgRouterThr:2040078 [router.py:message_loop():75] message_loop has been closed

metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/config.yaml

@@ -0,0 +1,156 @@
+task:
+  name: button-press-topdown-wall
+  task_name: ${name}
+  shape_meta:
+    obs:
+      point_cloud:
+        shape:
+        - 512
+        - 3
+        type: point_cloud
+      agent_pos:
+        shape:
+        - 9
+        type: low_dim
+    action:
+      shape:
+      - 4
+  env_runner:
+    _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner
+    eval_episodes: 20
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    fps: 10
+    n_envs: null
+    n_train: null
+    n_test: null
+    task_name: ${task_name}
+    device: ${training.device}
+    use_point_crop: ${policy.use_point_crop}
+  dataset:
+    _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset
+    zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-topdown-wall_expert.zarr
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    seed: 42
+    val_ratio: 0.02
+    max_train_episodes: 90
+name: train_dp3
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: metaworld_button-press-topdown-wall-dp3-1218
+horizon: 4
+n_obs_steps: 2
+n_action_steps: 3
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy_3d.policy.dp3.DP3
+  use_point_crop: true
+  condition_type: film
+  use_down_condition: true
+  use_mid_condition: true
+  use_up_condition: true
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  crop_shape:
+  - 80
+  - 80
+  encoder_output_dim: 64
+  horizon: ${horizon}
+  kernel_size: 5
+  n_action_steps: ${n_action_steps}
+  n_groups: 8
+  n_obs_steps: ${n_obs_steps}
+  noise_scheduler:
+    _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    clip_sample: true
+    set_alpha_to_one: true
+    steps_offset: 0
+    prediction_type: sample
+  num_inference_steps: 10
+  obs_as_global_cond: true
+  shape_meta: ${shape_meta}
+  use_pc_color: false
+  pointnet_type: pointnet
+  pointcloud_encoder_cfg:
+    in_channels: 3
+    out_channels: ${policy.encoder_output_dim}
+    use_layernorm: true
+    final_norm: layernorm
+    normal_channel: false
+ema:
+  _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 128
+  num_workers: 8
+  shuffle: true
+  pin_memory: true
+  persistent_workers: false
+val_dataloader:
+  batch_size: 128
+  num_workers: 8
+  shuffle: false
+  pin_memory: true
+  persistent_workers: false
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 0
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 3000
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 200
+  checkpoint_every: 200
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  group: ${exp_name}
+  id: null
+  mode: ''
+  name: ${training.seed}
+  project: dp3
+  resume: true
+  tags:
+  - dp3
+checkpoint:
+  save_ckpt: ''
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: true
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}

metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/overrides.yaml

@@ -0,0 +1,7 @@
+- task=metaworld_button-press-topdown-wall
+- training.debug=False
+- training.seed=0
+- training.device=cuda:0
+- exp_name=metaworld_button-press-topdown-wall-dp3-1218
+- logging.mode=
+- checkpoint.save_ckpt=

metaworld_button-press-topdown-wall-dp3-1218_seed0/eval_results.json

@@ -0,0 +1,9 @@
+{
+    "mean_traj_rewards": 262.50912456420383,
+    "mean_success_rates": 1.0,
+    "mean_durations": 4.872495170589536,
+    "std_durations": 0.07859229908783016,
+    "test_mean_score": 1.0,
+    "SR_test_L3": 1.0,
+    "SR_test_L5": 1.0
+}

metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/files/wandb-summary.json

@@ -0,0 +1 @@
+{"bc_loss":9.746736395754851e-07,"_timestamp":1.7345114860345306e+09,"global_step":44999,"train_loss":6.30829500399462e-07,"lr":0,"_wandb":{"runtime":7898},"mean_success_rates":1,"_runtime":7898.281424519,"test_mean_score":1,"train_action_mse_error":1.1380741852917708e-05,"SR_test_L3":1,"SR_test_L5":1,"epoch":2999,"mean_traj_rewards":201.4324005806066,"_step":44999}

metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log

@@ -0,0 +1,24 @@
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_setup.py:_flush():68] Configure stats pid to 2434667
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from environment variables
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log
+2024-12-18 14:33:07,732 INFO    MainThread:2434667 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug-internal.log
+2024-12-18 14:33:07,733 INFO    MainThread:2434667 [wandb_init.py:init():644] calling init triggers
+2024-12-18 14:33:07,733 INFO    MainThread:2434667 [wandb_init.py:init():650] wandb.init called with sweep_config: {}
+config: {'task': {'name': 'button-press-topdown-wall', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'button-press-topdown-wall', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_button-press-topdown-wall_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'button-press-topdown-wall', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_button-press-topdown-wall-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_button-press-topdown-wall-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/14.33.05_train_dp3_button-press-topdown-wall', 'wandb_name_base': '2024.12.18-14.33.05_train_dp3_button-press-topdown-wall'}}
+2024-12-18 14:33:07,733 INFO    MainThread:2434667 [wandb_init.py:init():680] starting backend
+2024-12-18 14:33:07,733 INFO    MainThread:2434667 [wandb_init.py:init():684] sending inform_init request
+2024-12-18 14:33:07,760 INFO    MainThread:2434667 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-12-18 14:33:07,761 INFO    MainThread:2434667 [wandb_init.py:init():697] backend started and connected
+2024-12-18 14:33:07,766 INFO    MainThread:2434667 [wandb_init.py:init():790] updated telemetry
+2024-12-18 14:33:07,921 INFO    MainThread:2434667 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout
+2024-12-18 14:33:09,409 INFO    MainThread:2434667 [wandb_init.py:init():874] starting run threads in backend
+2024-12-18 14:33:09,542 INFO    MainThread:2434667 [wandb_run.py:_console_start():2374] atexit reg
+2024-12-18 14:33:09,542 INFO    MainThread:2434667 [wandb_run.py:_redirect():2224] redirect: wrap_raw
+2024-12-18 14:33:09,543 INFO    MainThread:2434667 [wandb_run.py:_redirect():2289] Wrapping output streams.
+2024-12-18 14:33:09,543 INFO    MainThread:2434667 [wandb_run.py:_redirect():2314] Redirects installed.
+2024-12-18 14:33:09,545 INFO    MainThread:2434667 [wandb_init.py:init():916] run started, returning control to user process
+2024-12-18 14:33:09,546 INFO    MainThread:2434667 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0'}
+2024-12-18 16:44:46,043 WARNING MsgRouterThr:2434667 [router.py:message_loop():75] message_loop has been closed

metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_175628-z8y9ggjn/files/output.log

@@ -0,0 +1,131 @@
+Eval in Metaworld button-press-topdown-wall Pointcloud Env:   0%|                                                                                                               | 0/20 [00:00<?, ?it/s]wandb: WARNING Tried to log to step 42014 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42015 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42016 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42017 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42018 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42019 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42020 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42021 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42022 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42023 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42024 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42025 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42026 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42027 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Training epoch 2806:   0%|                                                                                                                                                      | 0/15 [00:00<?, ?it/s]wandb: WARNING Tried to log to step 42028 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+test_mean_score: 1.0
+mean_durations: 4.832249682955444
+std_durations: 0.018576822789998507
+wandb: WARNING Tried to log to step 42029 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42030 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42031 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42032 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42033 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42034 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42035 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42036 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42037 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42038 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42039 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42040 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42041 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42042 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42043 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42044 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42045 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42046 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42047 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42048 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42049 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42050 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42051 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42052 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42053 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42054 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42055 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42056 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42057 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42058 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42059 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42060 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42061 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42062 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42063 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42064 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42065 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42066 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42067 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42068 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42069 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42070 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42071 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42072 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42073 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42074 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42075 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42076 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42077 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42078 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42079 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42080 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42081 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42082 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42083 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42084 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42085 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42086 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42087 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42088 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42089 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42090 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42091 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42092 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42093 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42094 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42095 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42096 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42097 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42098 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42099 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42100 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42101 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42102 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+wandb: WARNING Tried to log to step 42103 that is less than the current step 45000. Steps must be monotonically increasing, so this data will be ignored. See https://wandb.me/define-metric to log data out of order.
+Traceback (most recent call last):                                                                                                                                                                     
+  File "train.py", line 506, in <module>
+    main()
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/main.py", line 90, in decorated_main
+    _run_hydra(
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 389, in _run_hydra
+    _run_app(
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 452, in _run_app
+    run_and_report(
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 213, in run_and_report
+    return func()
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 453, in <lambda>
+    lambda: hydra.run(
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/hydra.py", line 119, in run
+    ret = run_job(
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/core/utils.py", line 186, in run_job
+    ret.return_value = task_function(task_cfg)
+  File "train.py", line 503, in main
+    workspace.run()
+  File "train.py", line 186, in run
+    for batch_idx, batch in enumerate(tepoch):
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/tqdm/std.py", line 1181, in __iter__
+    for obj in iterable:
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 630, in __next__
+    data = self._next_data()
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1316, in _next_data
+    self._shutdown_workers()
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1441, in _shutdown_workers
+    w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL)
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/process.py", line 149, in join
+    res = self._popen.wait(timeout)
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/popen_fork.py", line 44, in wait
+    if not wait([self.sentinel], timeout):
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/connection.py", line 931, in wait
+    ready = selector.select(timeout)
+  File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/selectors.py", line 415, in select
+    fd_event_list = self._selector.poll(timeout)
+KeyboardInterrupt

metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/wandb-resume.json

@@ -0,0 +1 @@
+{"run_id": "z8y9ggjn"}

metaworld_button-press-wall-dp3-1218_seed0/.hydra/config.yaml

@@ -0,0 +1,156 @@
+task:
+  name: button-press-wall
+  task_name: ${name}
+  shape_meta:
+    obs:
+      point_cloud:
+        shape:
+        - 512
+        - 3
+        type: point_cloud
+      agent_pos:
+        shape:
+        - 9
+        type: low_dim
+    action:
+      shape:
+      - 4
+  env_runner:
+    _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner
+    eval_episodes: 20
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    fps: 10
+    n_envs: null
+    n_train: null
+    n_test: null
+    task_name: ${task_name}
+    device: ${training.device}
+    use_point_crop: ${policy.use_point_crop}
+  dataset:
+    _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset
+    zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-wall_expert.zarr
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    seed: 42
+    val_ratio: 0.02
+    max_train_episodes: 90
+name: train_dp3
+task_name: ${task.name}
+shape_meta: ${task.shape_meta}
+exp_name: metaworld_button-press-wall-dp3-1218
+horizon: 4
+n_obs_steps: 2
+n_action_steps: 3
+n_latency_steps: 0
+dataset_obs_steps: ${n_obs_steps}
+keypoint_visible_rate: 1.0
+obs_as_global_cond: true
+policy:
+  _target_: diffusion_policy_3d.policy.dp3.DP3
+  use_point_crop: true
+  condition_type: film
+  use_down_condition: true
+  use_mid_condition: true
+  use_up_condition: true
+  diffusion_step_embed_dim: 128
+  down_dims:
+  - 512
+  - 1024
+  - 2048
+  crop_shape:
+  - 80
+  - 80
+  encoder_output_dim: 64
+  horizon: ${horizon}
+  kernel_size: 5
+  n_action_steps: ${n_action_steps}
+  n_groups: 8
+  n_obs_steps: ${n_obs_steps}
+  noise_scheduler:
+    _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    clip_sample: true
+    set_alpha_to_one: true
+    steps_offset: 0
+    prediction_type: sample
+  num_inference_steps: 10
+  obs_as_global_cond: true
+  shape_meta: ${shape_meta}
+  use_pc_color: false
+  pointnet_type: pointnet
+  pointcloud_encoder_cfg:
+    in_channels: 3
+    out_channels: ${policy.encoder_output_dim}
+    use_layernorm: true
+    final_norm: layernorm
+    normal_channel: false
+ema:
+  _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 128
+  num_workers: 8
+  shuffle: true
+  pin_memory: true
+  persistent_workers: false
+val_dataloader:
+  batch_size: 128
+  num_workers: 8
+  shuffle: false
+  pin_memory: true
+  persistent_workers: false
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 0
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 3000
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 200
+  checkpoint_every: 200
+  val_every: 1
+  sample_every: 5
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  group: ${exp_name}
+  id: null
+  mode: ''
+  name: ${training.seed}
+  project: dp3
+  resume: true
+  tags:
+  - dp3
+checkpoint:
+  save_ckpt: ''
+  topk:
+    monitor_key: test_mean_score
+    mode: max
+    k: 1
+    format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
+  save_last_ckpt: true
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}