diff --git a/adroit_hammer-dp3-1218_seed0/wandb/debug.log b/adroit_hammer-dp3-1218_seed0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e82a9f34e0e74424329802e54e0e1f61c1bbc1f3 --- /dev/null +++ b/adroit_hammer-dp3-1218_seed0/wandb/debug.log @@ -0,0 +1,24 @@ +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_setup.py:_flush():68] Configure stats pid to 4158545 +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-19 02:22:47,801 INFO MainThread:4158545 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/logs/debug.log +2024-12-19 02:22:47,802 INFO MainThread:4158545 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/logs/debug-internal.log +2024-12-19 02:22:47,802 INFO MainThread:4158545 [wandb_init.py:init():644] calling init triggers +2024-12-19 02:22:47,802 INFO MainThread:4158545 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'adroit_hammer', 'task_name': 'hammer', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'hammer', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_hammer_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_hammer', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'exp_name': 'adroit_hammer-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [26]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_hammer-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.22.45_train_dp3_adroit_hammer', 'wandb_name_base': '2024.12.19-02.22.45_train_dp3_adroit_hammer'}} +2024-12-19 02:22:47,802 INFO MainThread:4158545 [wandb_init.py:init():680] starting backend +2024-12-19 02:22:47,802 INFO MainThread:4158545 [wandb_init.py:init():684] sending inform_init request +2024-12-19 02:22:47,830 INFO MainThread:4158545 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-19 02:22:47,830 INFO MainThread:4158545 [wandb_init.py:init():697] backend started and connected +2024-12-19 02:22:47,837 INFO MainThread:4158545 [wandb_init.py:init():790] updated telemetry +2024-12-19 02:22:48,020 INFO MainThread:4158545 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-19 02:22:49,553 INFO MainThread:4158545 [wandb_init.py:init():874] starting run threads in backend +2024-12-19 02:22:49,672 INFO MainThread:4158545 [wandb_run.py:_console_start():2374] atexit reg +2024-12-19 02:22:49,673 INFO MainThread:4158545 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-19 02:22:49,673 INFO MainThread:4158545 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-19 02:22:49,673 INFO MainThread:4158545 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-19 02:22:49,675 INFO MainThread:4158545 [wandb_init.py:init():916] run started, returning control to user process +2024-12-19 02:22:49,676 INFO MainThread:4158545 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0'} +2024-12-19 03:42:27,921 WARNING MsgRouterThr:4158545 [router.py:message_loop():75] message_loop has been closed diff --git a/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/config.yaml b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b135a39bca176f8a070980d4195e0ccbc9ecece --- /dev/null +++ b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/config.yaml @@ -0,0 +1,244 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: adroit_hammer-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: adroit_hammer-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.19/02.22.45_train_dp3_adroit_hammer + wandb_name_base: 2024.12.19-02.22.45_train_dp3_adroit_hammer +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 26 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 26 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/adroit_hammer_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner + eval_episodes: 20 + fps: 10 + max_steps: 300 + n_action_steps: 3 + n_obs_steps: 2 + render_size: 84 + task_name: hammer + use_point_crop: true + image_shape: + - 3 + - 84 + - 84 + name: adroit_hammer + shape_meta: + action: + shape: + - 26 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: hammer +task_name: + value: adroit_hammer +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_17606_ff035320c5ebeb1276dd.mp4 b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_17606_ff035320c5ebeb1276dd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..010bdc358b676d43ac54ed97be53dd9e16a7128a Binary files /dev/null and b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_17606_ff035320c5ebeb1276dd.mp4 differ diff --git a/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_4806_08c8708afadd378013c7.mp4 b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_4806_08c8708afadd378013c7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ab7ca06d53ee2520a8b2b02fac30c431beda4a08 Binary files /dev/null and b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_4806_08c8708afadd378013c7.mp4 differ diff --git a/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_9606_5220e8762798d31f6a27.mp4 b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_9606_5220e8762798d31f6a27.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..980ae2d6ed11f167101c7f5c3ccb3240b3550b3b Binary files /dev/null and b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/media/videos/sim_video_eval_9606_5220e8762798d31f6a27.mp4 differ diff --git a/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/wandb-metadata.json b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f86737bb38416f1b8882b5536dc277b0482650c3 --- /dev/null +++ b/adroit_hammer-dp3-1218_seed0/wandb/run-20241219_022247-cjrkvstt/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.17", + "python": "CPython 3.8.20", + "startedAt": "2024-12-18T18:22:47.830749Z", + "args": [ + "--config-name=dp3.yaml", + "task=adroit_hammer", + "hydra.run.dir=data/outputs/adroit_hammer-dp3-1218_seed0", + "training.debug=False", + "training.seed=0", + "training.device=cuda:0", + "exp_name=adroit_hammer-dp3-1218", + "logging.mode=online", + "checkpoint.save_ckpt=True" + ], + "program": "train.py", + "codePath": "3D-Diffusion-Policy/train.py", + "git": { + "remote": "git@github.com:YanjieZe/3D-Diffusion-Policy.git", + "commit": "3153fb32c17308fc828015156480b77f6b2e666a" + }, + "email": "chenhaojun001@gmail.com", + "root": "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_hammer-dp3-1218_seed0", + "host": "Server-3090", + "executable": "/home/chenhaojun/miniconda3/envs/dp3/bin/python", + "codePathLocal": "train.py", + "cpu_count": 128, + "cpu_count_logical": 256, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "943412031488", + "used": "837418799104" + } + }, + "memory": { + "total": "1081980669952" + }, + "cpu": { + "count": 128, + "countLogical": 256 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.5" +} \ No newline at end of file diff --git a/adroit_pen-dp3-1218_seed0/.hydra/hydra.yaml b/adroit_pen-dp3-1218_seed0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08bafb86ee93668616e05fd8540a7a9c3105f278 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: data/outputs/adroit_pen-dp3-1218_seed0 + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=data/outputs/adroit_pen-dp3-1218_seed0 + - hydra.mode=RUN + task: + - task=adroit_pen + - training.debug=False + - training.seed=0 + - training.device=cuda:0 + - exp_name=adroit_pen-dp3-1218 + - logging.mode=online + - checkpoint.save_ckpt=True + job: + name: train + chdir: null + override_dirname: checkpoint.save_ckpt=True,exp_name=adroit_pen-dp3-1218,logging.mode=online,task=adroit_pen,training.debug=False,training.device=cuda:0,training.seed=0 + id: ??? + num: ??? + config_name: dp3.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.2.0 + version_base: '1.2' + cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0 + choices: + task: adroit_pen + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/adroit_pen-dp3-1218_seed0/wandb/debug-internal.log b/adroit_pen-dp3-1218_seed0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..42c8bc4a87755b4c77d0ce4be0de0139fcf92afc --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-12-19T02:23:11.067012008+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-19T02:23:11.067039698+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-core.log"} +{"time":"2024-12-19T02:23:11.287533236+08:00","level":"INFO","msg":"created new stream","id":"wnx82u0h"} +{"time":"2024-12-19T02:23:11.287579885+08:00","level":"INFO","msg":"stream: started","id":"wnx82u0h"} +{"time":"2024-12-19T02:23:11.287650415+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"wnx82u0h"} +{"time":"2024-12-19T02:23:11.287714795+08:00","level":"INFO","msg":"handler: started","stream_id":"wnx82u0h"} +{"time":"2024-12-19T02:23:11.287915284+08:00","level":"INFO","msg":"sender: started","stream_id":"wnx82u0h"} +{"time":"2024-12-19T02:23:13.851169857+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-19T03:39:20.847103676+08:00","level":"INFO","msg":"stream: closing","id":"wnx82u0h"} +{"time":"2024-12-19T03:39:20.847167426+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-19T03:39:20.848587386+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-19T03:39:23.680955487+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-19T03:39:26.446790779+08:00","level":"INFO","msg":"handler: closed","stream_id":"wnx82u0h"} +{"time":"2024-12-19T03:39:26.446866839+08:00","level":"INFO","msg":"sender: closed","stream_id":"wnx82u0h"} +{"time":"2024-12-19T03:39:26.446853389+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"wnx82u0h"} +{"time":"2024-12-19T03:39:26.447021069+08:00","level":"INFO","msg":"stream: closed","id":"wnx82u0h"} diff --git a/adroit_pen-dp3-1218_seed0/wandb/debug.log b/adroit_pen-dp3-1218_seed0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..42c49df5771a6ed49d1b5740c357c424b1b23490 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/debug.log @@ -0,0 +1,24 @@ +2024-12-19 02:23:11,035 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Configure stats pid to 4160108 +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-internal.log +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:init():644] calling init triggers +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'adroit_pen', 'task_name': 'pen', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'pen', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_pen_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_pen', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'exp_name': 'adroit_pen-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_pen-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen', 'wandb_name_base': '2024.12.19-02.23.08_train_dp3_adroit_pen'}} +2024-12-19 02:23:11,037 INFO MainThread:4160108 [wandb_init.py:init():680] starting backend +2024-12-19 02:23:11,037 INFO MainThread:4160108 [wandb_init.py:init():684] sending inform_init request +2024-12-19 02:23:11,063 INFO MainThread:4160108 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-19 02:23:11,064 INFO MainThread:4160108 [wandb_init.py:init():697] backend started and connected +2024-12-19 02:23:11,069 INFO MainThread:4160108 [wandb_init.py:init():790] updated telemetry +2024-12-19 02:23:11,259 INFO MainThread:4160108 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-19 02:23:13,842 INFO MainThread:4160108 [wandb_init.py:init():874] starting run threads in backend +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_console_start():2374] atexit reg +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-19 02:23:13,969 INFO MainThread:4160108 [wandb_init.py:init():916] run started, returning control to user process +2024-12-19 02:23:13,969 INFO MainThread:4160108 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0'} +2024-12-19 03:39:20,847 WARNING MsgRouterThr:4160108 [router.py:message_loop():75] message_loop has been closed diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/config.yaml b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d38326bbcf2f21022314e77ed1603aebcaa70d9 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/config.yaml @@ -0,0 +1,244 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: adroit_pen-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: adroit_pen-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen + wandb_name_base: 2024.12.19-02.23.08_train_dp3_adroit_pen +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 24 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 24 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/adroit_pen_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner + eval_episodes: 20 + fps: 10 + max_steps: 300 + n_action_steps: 3 + n_obs_steps: 2 + render_size: 84 + task_name: pen + use_point_crop: true + image_shape: + - 3 + - 84 + - 84 + name: adroit_pen + shape_meta: + action: + shape: + - 24 + obs: + agent_pos: + shape: + - 24 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: pen +task_name: + value: adroit_pen +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_12806_389e49055054822347b4.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_12806_389e49055054822347b4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3a71eb089ddb38e0d7f109b995e16280e2fcf32a Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_12806_389e49055054822347b4.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_14406_1f51387baf0f0e8dec1e.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_14406_1f51387baf0f0e8dec1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f8292ed30176c11004a706d758c902e61e7e4817 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_14406_1f51387baf0f0e8dec1e.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_1606_52c5e802c44280121992.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_1606_52c5e802c44280121992.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..62e778f25190916f2baf4bb78a3237d1f695a0a0 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_1606_52c5e802c44280121992.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_17606_07176654aa7bd93f90fd.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_17606_07176654aa7bd93f90fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..107bbaaad6498c3a71b70d8ffa9bd49d30eecf51 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_17606_07176654aa7bd93f90fd.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_19206_5b1f4130467fbb14fe02.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_19206_5b1f4130467fbb14fe02.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d9c89abcda2f25054770a1be7af3136c88b1bd5 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_19206_5b1f4130467fbb14fe02.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_20806_9ed3d6dcd9d5832d0636.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_20806_9ed3d6dcd9d5832d0636.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..71f20d2ec53889512035e38446e101a58271252e Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_20806_9ed3d6dcd9d5832d0636.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_22406_84ba9495019a4765c1ed.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_22406_84ba9495019a4765c1ed.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8dab4a401e21869856be7a5c1026a968e179bf5 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_22406_84ba9495019a4765c1ed.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6406_60646a2588dbb484e91e.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6406_60646a2588dbb484e91e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4773894c1a06c144ba46be1c0b7f09e5558c0e91 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6406_60646a2588dbb484e91e.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6_563241821d690ce33091.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6_563241821d690ce33091.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..46605837a9e7f98635909ae9af121f54abe23894 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_6_563241821d690ce33091.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_8006_14377171a681f39a428b.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_8006_14377171a681f39a428b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1c3c4073e040caa9f10ebdcbd80f898fb101800 Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_8006_14377171a681f39a428b.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_9606_40c4de222a5a2e422a57.mp4 b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_9606_40c4de222a5a2e422a57.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..837bc611d19d8cfa6952898dbe9fe1ea7cba10ae Binary files /dev/null and b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/media/videos/sim_video_eval_9606_40c4de222a5a2e422a57.mp4 differ diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/output.log b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..81e18482f90debfdfe7d5bb89f114c83e1170632 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/output.log @@ -0,0 +1,16 @@ + +test_mean_score: 0.0 +test_mean_score: 0.35 +test_mean_score: 0.25 +test_mean_score: 0.15 +test_mean_score: 0.5 +test_mean_score: 0.35 +test_mean_score: 0.35 +test_mean_score: 0.3 +test_mean_score: 0.3 +test_mean_score: 0.425 +test_mean_score: 0.5 +test_mean_score: 0.45 +test_mean_score: 0.25 +test_mean_score: 0.25 +test_mean_score: 0.45 diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/requirements.txt b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4932de6b67b7954a352c7097d35162e7d60ed6c9 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/files/requirements.txt @@ -0,0 +1,175 @@ +diffusion_policy_3d==0.0.0 +certifi==2024.12.14 +PyYAML==6.0.2 +dash-table==5.0.0 +dm_control==1.0.23 +einops==0.4.1 +ptyprocess==0.7.0 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +nbformat==5.10.4 +decorator==4.4.2 +importlib_metadata==8.5.0 +numcodecs==0.12.1 +requests==2.32.3 +ipywidgets==8.1.5 +urllib3==2.2.3 +platformdirs==4.3.6 +imageio-ffmpeg==0.5.1 +dash==2.18.2 +Cython==0.29.35 +mujoco-py==2.1.2.14 +mpmath==1.3.0 +typing_extensions==4.12.2 +dm-tree==0.1.8 +imageio==2.35.1 +dill==0.3.5.1 +joblib==1.4.2 +attrs==24.3.0 +torchvision==0.19.1+cu121 +kiwisolver==1.4.7 +numpy==1.23.5 +pillow==10.2.0 +nvidia-cuda-cupti-cu12==12.1.105 +pip==24.2 +llvmlite==0.39.1 +tenacity==9.0.0 +scipy==1.10.1 +charset-normalizer==3.4.0 +matplotlib==3.7.5 +jupyterlab_widgets==3.0.13 +patchelf==0.17.2.0 +wcwidth==0.2.13 +pexpect==4.9.0 +plotly==5.24.1 +ipdb==0.13.13 +referencing==0.35.1 +mujoco==3.2.3 +portalocker==3.0.0 +jedi==0.19.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +nvidia-nvtx-cu12==12.1.105 +python-dateutil==2.9.0.post0 +packaging==24.2 +click==8.1.7 +pycparser==2.22 +blinker==1.8.2 +sympy==1.13.1 +Werkzeug==3.0.6 +six==1.17.0 +stack-data==0.6.3 +gpustat==1.1.1 +threadpoolctl==3.5.0 +traitlets==5.14.3 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.1.105 +zarr==2.12.0 +pandas==2.0.3 +wheel==0.44.0 +nvidia-nccl-cu12==2.20.5 +comm==0.2.2 +proglog==0.1.10 +parso==0.8.4 +open3d==0.18.0 +nvidia-cusparse-cu12==12.1.0.106 +antlr4-python3-runtime==4.9.3 +fvcore==0.1.5.post20221221 +natsort==8.4.0 +cloudpickle==3.1.0 +ConfigArgParse==1.7 +fonttools==4.55.3 +etils==1.3.0 +torchaudio==2.4.1+cu121 +pickleshare==0.7.5 +eval_type_backport==0.2.0 +executing==2.1.0 +addict==2.4.0 +protobuf==5.29.1 +setproctitle==1.3.4 +Flask==3.0.3 +filelock==3.13.1 +regex==2024.11.6 +PyOpenGL==3.1.7 +nvidia-curand-cu12==10.3.2.106 +matplotlib-inline==0.1.7 +transforms3d==0.4.2 +pydantic==2.10.3 +cffi==1.17.1 +rpds-py==0.20.1 +diffusers==0.11.1 +docker-pycreds==0.4.0 +pyparsing==3.1.4 +absl-py==2.1.0 +nvidia-cufft-cu12==11.0.2.54 +contourpy==1.1.1 +fastjsonschema==2.21.1 +pkgutil_resolve_name==1.3.10 +dash-html-components==2.0.0 +asciitree==0.3.3 +fasteners==0.19 +glfw==2.8.0 +omegaconf==2.3.0 +asttokens==3.0.0 +moviepy==1.0.3 +GitPython==3.1.43 +opencv-python==4.10.0.84 +zipp==3.20.2 +tabulate==0.9.0 +iopath==0.1.10 +labmaze==1.0.6 +numba==0.56.4 +nvidia-cuda-runtime-cu12==12.1.105 +dm-env==1.6 +cycler==0.12.1 +dash-core-components==2.0.0 +networkx==3.0 +Pygments==2.18.0 +retrying==1.3.4 +jsonschema-specifications==2023.12.1 +setuptools==59.5.0 +av==12.3.0 +itsdangerous==2.2.0 +wandb==0.19.1 +nest-asyncio==1.6.0 +triton==3.0.0 +nvidia-ml-py==12.560.30 +nvidia-cuda-nvrtc-cu12==12.1.105 +tomli==2.2.1 +tqdm==4.67.1 +termcolor==2.4.0 +hydra-core==1.2.0 +smmap==5.0.1 +ipython==8.12.3 +pyquaternion==0.9.9 +idna==3.10 +yacs==0.1.8 +tzdata==2024.2 +torch==2.4.1+cu121 +widgetsnbextension==4.0.13 +scikit-learn==1.3.2 +sapien==2.2.1 +nvidia-cublas-cu12==12.1.3.1 +huggingface-hub==0.21.0 +gitdb==4.0.11 +importlib_resources==6.4.5 +pydantic_core==2.27.1 +jupyter_core==5.7.2 +sentry-sdk==2.19.2 +nvidia-cudnn-cu12==9.1.0.70 +fsspec==2024.2.0 +psutil==6.1.0 +pytz==2024.2 +backcall==0.2.0 +nvidia-cusolver-cu12==11.4.5.107 +lxml==5.3.0 +blessed==1.20.0 +pure_eval==0.2.3 +diffusion_policy_3d==0.0.0 +dexart==0.1.0 +gym==0.21.0 +metaworld==0.1.0 +mj-envs==1.0.0 +mjrl==1.0.0 +pytorch3d==0.7.6 +diffusion_policy_3d==0.0.0 diff --git a/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..42c49df5771a6ed49d1b5740c357c424b1b23490 --- /dev/null +++ b/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log @@ -0,0 +1,24 @@ +2024-12-19 02:23:11,035 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Configure stats pid to 4160108 +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug.log +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0/wandb/run-20241219_022311-wnx82u0h/logs/debug-internal.log +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:init():644] calling init triggers +2024-12-19 02:23:11,036 INFO MainThread:4160108 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'adroit_pen', 'task_name': 'pen', 'image_shape': [3, 84, 84], 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.adroit_runner.AdroitRunner', 'eval_episodes': 20, 'max_steps': 300, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'task_name': 'pen', 'render_size': 84, 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.adroit_dataset.AdroitDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/adroit_pen_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'adroit_pen', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'exp_name': 'adroit_pen-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [24], 'type': 'low_dim'}}, 'action': {'shape': [24]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'adroit_pen-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.19/02.23.08_train_dp3_adroit_pen', 'wandb_name_base': '2024.12.19-02.23.08_train_dp3_adroit_pen'}} +2024-12-19 02:23:11,037 INFO MainThread:4160108 [wandb_init.py:init():680] starting backend +2024-12-19 02:23:11,037 INFO MainThread:4160108 [wandb_init.py:init():684] sending inform_init request +2024-12-19 02:23:11,063 INFO MainThread:4160108 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-19 02:23:11,064 INFO MainThread:4160108 [wandb_init.py:init():697] backend started and connected +2024-12-19 02:23:11,069 INFO MainThread:4160108 [wandb_init.py:init():790] updated telemetry +2024-12-19 02:23:11,259 INFO MainThread:4160108 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-19 02:23:13,842 INFO MainThread:4160108 [wandb_init.py:init():874] starting run threads in backend +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_console_start():2374] atexit reg +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-19 02:23:13,966 INFO MainThread:4160108 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-19 02:23:13,969 INFO MainThread:4160108 [wandb_init.py:init():916] run started, returning control to user process +2024-12-19 02:23:13,969 INFO MainThread:4160108 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/adroit_pen-dp3-1218_seed0'} +2024-12-19 03:39:20,847 WARNING MsgRouterThr:4160108 [router.py:message_loop():75] message_loop has been closed diff --git a/dexart_bucket-dp3-1219_seed0/.hydra/overrides.yaml b/dexart_bucket-dp3-1219_seed0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53d72be5075f81efab30a00f02d04240b13f803b --- /dev/null +++ b/dexart_bucket-dp3-1219_seed0/.hydra/overrides.yaml @@ -0,0 +1,7 @@ +- task=dexart_bucket +- training.debug=False +- training.seed=0 +- training.device=cuda:0 +- exp_name=dexart_bucket-dp3-1219 +- logging.mode=online +- checkpoint.save_ckpt=True diff --git a/dexart_bucket-dp3-1219_seed0/train.log b/dexart_bucket-dp3-1219_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2d377fd4771b3f0e240ba927268659bce846ebe1 --- /dev/null +++ b/dexart_bucket-dp3-1219_seed0/train.log @@ -0,0 +1 @@ +[2024-12-19 16:44:56,429][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.551390e+08 diff --git a/metaworld_assembly-dp3-1218_seed0/wandb/debug-internal.log b/metaworld_assembly-dp3-1218_seed0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..deeadd8d9e0e8c969b34558d4bef7789311472af --- /dev/null +++ b/metaworld_assembly-dp3-1218_seed0/wandb/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-12-18T14:28:52.397338454+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-18T14:28:52.397351034+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug-core.log"} +{"time":"2024-12-18T14:28:52.609650827+08:00","level":"INFO","msg":"created new stream","id":"63943iru"} +{"time":"2024-12-18T14:28:52.609680037+08:00","level":"INFO","msg":"stream: started","id":"63943iru"} +{"time":"2024-12-18T14:28:52.609755478+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"63943iru"} +{"time":"2024-12-18T14:28:52.609795528+08:00","level":"INFO","msg":"handler: started","stream_id":"63943iru"} +{"time":"2024-12-18T14:28:52.611313401+08:00","level":"INFO","msg":"sender: started","stream_id":"63943iru"} +{"time":"2024-12-18T14:28:55.113722346+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-18T16:42:47.860321056+08:00","level":"INFO","msg":"stream: closing","id":"63943iru"} +{"time":"2024-12-18T16:42:47.860366545+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-18T16:42:47.861825165+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-18T16:42:51.203826373+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-18T16:42:54.393251212+08:00","level":"INFO","msg":"handler: closed","stream_id":"63943iru"} +{"time":"2024-12-18T16:42:54.393317331+08:00","level":"INFO","msg":"sender: closed","stream_id":"63943iru"} +{"time":"2024-12-18T16:42:54.393312931+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"63943iru"} +{"time":"2024-12-18T16:42:54.393483768+08:00","level":"INFO","msg":"stream: closed","id":"63943iru"} diff --git a/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/config.yaml b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41bce6e5bdbdd25ad4708ef58dd4aafdf4d29417 --- /dev/null +++ b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/config.yaml @@ -0,0 +1,242 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: metaworld_assembly-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: metaworld_assembly-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.18/14.28.49_train_dp3_assembly + wandb_name_base: 2024.12.18-14.28.49_train_dp3_assembly +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/metaworld_assembly_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + device: cuda:0 + eval_episodes: 20 + fps: 10 + n_action_steps: 3 + n_envs: null + n_obs_steps: 2 + n_test: null + n_train: null + task_name: assembly + use_point_crop: true + name: assembly + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: train_dp3 +task_name: + value: assembly +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/wandb-summary.json b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..441e8f6b542219124d19e898fe2ee85f7a2a962b --- /dev/null +++ b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/files/wandb-summary.json @@ -0,0 +1 @@ +{"lr":0,"epoch":2999,"SR_test_L5":1,"_wandb":{"runtime":8035},"mean_traj_rewards":30884.26171460612,"mean_success_rates":1,"_runtime":8035.464377365,"global_step":44999,"bc_loss":2.0856390392509638e-07,"_step":44999,"train_action_mse_error":4.6260427666311443e-07,"SR_test_L3":1,"_timestamp":1.734511367852143e+09,"train_loss":2.420270874381458e-07,"test_mean_score":1} \ No newline at end of file diff --git a/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cecc352191cafe3e6cddb7628c0be5a849d75a30 --- /dev/null +++ b/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log @@ -0,0 +1,24 @@ +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_setup.py:_flush():68] Configure stats pid to 2428026 +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug.log +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0/wandb/run-20241218_142852-63943iru/logs/debug-internal.log +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:init():644] calling init triggers +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'assembly', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'assembly', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_assembly_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'assembly', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_assembly-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_assembly-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/14.28.49_train_dp3_assembly', 'wandb_name_base': '2024.12.18-14.28.49_train_dp3_assembly'}} +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:init():680] starting backend +2024-12-18 14:28:52,369 INFO MainThread:2428026 [wandb_init.py:init():684] sending inform_init request +2024-12-18 14:28:52,395 INFO MainThread:2428026 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 14:28:52,395 INFO MainThread:2428026 [wandb_init.py:init():697] backend started and connected +2024-12-18 14:28:52,400 INFO MainThread:2428026 [wandb_init.py:init():790] updated telemetry +2024-12-18 14:28:52,577 INFO MainThread:2428026 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 14:28:55,105 INFO MainThread:2428026 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 14:28:55,225 INFO MainThread:2428026 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 14:28:55,225 INFO MainThread:2428026 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 14:28:55,226 INFO MainThread:2428026 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 14:28:55,226 INFO MainThread:2428026 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 14:28:55,228 INFO MainThread:2428026 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 14:28:55,228 INFO MainThread:2428026 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_assembly-dp3-1218_seed0'} +2024-12-18 16:42:47,860 WARNING MsgRouterThr:2428026 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_bin-picking-dp3-1218_seed0/train.log b/metaworld_bin-picking-dp3-1218_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b25de4e036b84a2a9f0eea172b2dc898f8f4ae07 --- /dev/null +++ b/metaworld_bin-picking-dp3-1218_seed0/train.log @@ -0,0 +1 @@ +[2024-12-18 20:53:06,678][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 diff --git a/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/files/requirements.txt b/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4932de6b67b7954a352c7097d35162e7d60ed6c9 --- /dev/null +++ b/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/files/requirements.txt @@ -0,0 +1,175 @@ +diffusion_policy_3d==0.0.0 +certifi==2024.12.14 +PyYAML==6.0.2 +dash-table==5.0.0 +dm_control==1.0.23 +einops==0.4.1 +ptyprocess==0.7.0 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +nbformat==5.10.4 +decorator==4.4.2 +importlib_metadata==8.5.0 +numcodecs==0.12.1 +requests==2.32.3 +ipywidgets==8.1.5 +urllib3==2.2.3 +platformdirs==4.3.6 +imageio-ffmpeg==0.5.1 +dash==2.18.2 +Cython==0.29.35 +mujoco-py==2.1.2.14 +mpmath==1.3.0 +typing_extensions==4.12.2 +dm-tree==0.1.8 +imageio==2.35.1 +dill==0.3.5.1 +joblib==1.4.2 +attrs==24.3.0 +torchvision==0.19.1+cu121 +kiwisolver==1.4.7 +numpy==1.23.5 +pillow==10.2.0 +nvidia-cuda-cupti-cu12==12.1.105 +pip==24.2 +llvmlite==0.39.1 +tenacity==9.0.0 +scipy==1.10.1 +charset-normalizer==3.4.0 +matplotlib==3.7.5 +jupyterlab_widgets==3.0.13 +patchelf==0.17.2.0 +wcwidth==0.2.13 +pexpect==4.9.0 +plotly==5.24.1 +ipdb==0.13.13 +referencing==0.35.1 +mujoco==3.2.3 +portalocker==3.0.0 +jedi==0.19.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +nvidia-nvtx-cu12==12.1.105 +python-dateutil==2.9.0.post0 +packaging==24.2 +click==8.1.7 +pycparser==2.22 +blinker==1.8.2 +sympy==1.13.1 +Werkzeug==3.0.6 +six==1.17.0 +stack-data==0.6.3 +gpustat==1.1.1 +threadpoolctl==3.5.0 +traitlets==5.14.3 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.1.105 +zarr==2.12.0 +pandas==2.0.3 +wheel==0.44.0 +nvidia-nccl-cu12==2.20.5 +comm==0.2.2 +proglog==0.1.10 +parso==0.8.4 +open3d==0.18.0 +nvidia-cusparse-cu12==12.1.0.106 +antlr4-python3-runtime==4.9.3 +fvcore==0.1.5.post20221221 +natsort==8.4.0 +cloudpickle==3.1.0 +ConfigArgParse==1.7 +fonttools==4.55.3 +etils==1.3.0 +torchaudio==2.4.1+cu121 +pickleshare==0.7.5 +eval_type_backport==0.2.0 +executing==2.1.0 +addict==2.4.0 +protobuf==5.29.1 +setproctitle==1.3.4 +Flask==3.0.3 +filelock==3.13.1 +regex==2024.11.6 +PyOpenGL==3.1.7 +nvidia-curand-cu12==10.3.2.106 +matplotlib-inline==0.1.7 +transforms3d==0.4.2 +pydantic==2.10.3 +cffi==1.17.1 +rpds-py==0.20.1 +diffusers==0.11.1 +docker-pycreds==0.4.0 +pyparsing==3.1.4 +absl-py==2.1.0 +nvidia-cufft-cu12==11.0.2.54 +contourpy==1.1.1 +fastjsonschema==2.21.1 +pkgutil_resolve_name==1.3.10 +dash-html-components==2.0.0 +asciitree==0.3.3 +fasteners==0.19 +glfw==2.8.0 +omegaconf==2.3.0 +asttokens==3.0.0 +moviepy==1.0.3 +GitPython==3.1.43 +opencv-python==4.10.0.84 +zipp==3.20.2 +tabulate==0.9.0 +iopath==0.1.10 +labmaze==1.0.6 +numba==0.56.4 +nvidia-cuda-runtime-cu12==12.1.105 +dm-env==1.6 +cycler==0.12.1 +dash-core-components==2.0.0 +networkx==3.0 +Pygments==2.18.0 +retrying==1.3.4 +jsonschema-specifications==2023.12.1 +setuptools==59.5.0 +av==12.3.0 +itsdangerous==2.2.0 +wandb==0.19.1 +nest-asyncio==1.6.0 +triton==3.0.0 +nvidia-ml-py==12.560.30 +nvidia-cuda-nvrtc-cu12==12.1.105 +tomli==2.2.1 +tqdm==4.67.1 +termcolor==2.4.0 +hydra-core==1.2.0 +smmap==5.0.1 +ipython==8.12.3 +pyquaternion==0.9.9 +idna==3.10 +yacs==0.1.8 +tzdata==2024.2 +torch==2.4.1+cu121 +widgetsnbextension==4.0.13 +scikit-learn==1.3.2 +sapien==2.2.1 +nvidia-cublas-cu12==12.1.3.1 +huggingface-hub==0.21.0 +gitdb==4.0.11 +importlib_resources==6.4.5 +pydantic_core==2.27.1 +jupyter_core==5.7.2 +sentry-sdk==2.19.2 +nvidia-cudnn-cu12==9.1.0.70 +fsspec==2024.2.0 +psutil==6.1.0 +pytz==2024.2 +backcall==0.2.0 +nvidia-cusolver-cu12==11.4.5.107 +lxml==5.3.0 +blessed==1.20.0 +pure_eval==0.2.3 +diffusion_policy_3d==0.0.0 +dexart==0.1.0 +gym==0.21.0 +metaworld==0.1.0 +mj-envs==1.0.0 +mjrl==1.0.0 +pytorch3d==0.7.6 +diffusion_policy_3d==0.0.0 diff --git a/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log b/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..35b31c1e04acc7e71e4b3894816073f81fa436c6 --- /dev/null +++ b/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log @@ -0,0 +1,24 @@ +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_setup.py:_flush():68] Configure stats pid to 2962987 +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug.log +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0/wandb/run-20241218_205314-e0v576zv/logs/debug-internal.log +2024-12-18 20:53:14,968 INFO MainThread:2962987 [wandb_init.py:init():644] calling init triggers +2024-12-18 20:53:14,969 INFO MainThread:2962987 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'bin-picking', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'bin-picking', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_bin-picking_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'bin-picking', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_bin-picking-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_bin-picking-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/20.53.12_train_dp3_bin-picking', 'wandb_name_base': '2024.12.18-20.53.12_train_dp3_bin-picking'}} +2024-12-18 20:53:14,969 INFO MainThread:2962987 [wandb_init.py:init():680] starting backend +2024-12-18 20:53:14,969 INFO MainThread:2962987 [wandb_init.py:init():684] sending inform_init request +2024-12-18 20:53:14,996 INFO MainThread:2962987 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 20:53:14,997 INFO MainThread:2962987 [wandb_init.py:init():697] backend started and connected +2024-12-18 20:53:15,002 INFO MainThread:2962987 [wandb_init.py:init():790] updated telemetry +2024-12-18 20:53:15,163 INFO MainThread:2962987 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 20:53:17,822 INFO MainThread:2962987 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 20:53:17,936 INFO MainThread:2962987 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 20:53:17,937 INFO MainThread:2962987 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 20:53:17,937 INFO MainThread:2962987 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 20:53:17,937 INFO MainThread:2962987 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 20:53:17,939 INFO MainThread:2962987 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 20:53:17,939 INFO MainThread:2962987 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_bin-picking-dp3-1218_seed0'} +2024-12-19 00:26:54,104 WARNING MsgRouterThr:2962987 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_box-close-dp3-1218_seed0/.hydra/hydra.yaml b/metaworld_box-close-dp3-1218_seed0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e25a28088155533f7bb9a4a1d61468cc4d0faf2f --- /dev/null +++ b/metaworld_box-close-dp3-1218_seed0/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: data/outputs/metaworld_box-close-dp3-1218_seed0 + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=data/outputs/metaworld_box-close-dp3-1218_seed0 + - hydra.mode=RUN + task: + - task=metaworld_box-close + - training.debug=False + - training.seed=0 + - training.device=cuda:0 + - exp_name=metaworld_box-close-dp3-1218 + - logging.mode=online + - checkpoint.save_ckpt=True + job: + name: train + chdir: null + override_dirname: checkpoint.save_ckpt=True,exp_name=metaworld_box-close-dp3-1218,logging.mode=online,task=metaworld_box-close,training.debug=False,training.device=cuda:0,training.seed=0 + id: ??? + num: ??? + config_name: dp3.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.2.0 + version_base: '1.2' + cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_box-close-dp3-1218_seed0 + choices: + task: metaworld_box-close + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/output.log b/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..61db756b4cd4c4389f6089886a963101efa899e0 --- /dev/null +++ b/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/output.log @@ -0,0 +1,46 @@ + +test_mean_score: 0.0 +mean_durations: 4.828114485740661 +std_durations: 0.01738945887153302 +test_mean_score: 0.65 +mean_durations: 4.873560278303922 +std_durations: 0.019777209321623128 +test_mean_score: 0.0 +mean_durations: 4.854758024774492 +std_durations: 0.007352221929266012 +test_mean_score: 0.5 +mean_durations: 4.845281571056693 +std_durations: 0.008404259880539429 +test_mean_score: 0.6 +mean_durations: 4.851413570158184 +std_durations: 0.005525312932477265 +test_mean_score: 0.45 +mean_durations: 4.8451534550637 +std_durations: 0.005656914477641985 +test_mean_score: 0.55 +mean_durations: 4.850002092681825 +std_durations: 0.003998623920949613 +test_mean_score: 0.65 +mean_durations: 4.8544698669575155 +std_durations: 0.004563551825275274 +test_mean_score: 0.3 +mean_durations: 4.8519475953653455 +std_durations: 0.005867139433633655 +test_mean_score: 0.55 +mean_durations: 4.849922423437238 +std_durations: 0.024024316288814206 +test_mean_score: 0.45 +mean_durations: 4.851007116865366 +std_durations: 0.009671100741088231 +test_mean_score: 0.5 +mean_durations: 4.842636080086232 +std_durations: 0.0025684137610381937 +test_mean_score: 0.35 +mean_durations: 4.842003949917853 +std_durations: 0.004577108168226907 +test_mean_score: 0.2 +mean_durations: 4.850726788770407 +std_durations: 0.0029142146415510766 +test_mean_score: 0.45 +mean_durations: 4.84709279127419 +std_durations: 0.003142567013560075 diff --git a/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/wandb-summary.json b/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c9ebbee65066fe5aa070f0f7e7c2c420f062f101 --- /dev/null +++ b/metaworld_box-close-dp3-1218_seed0/wandb/run-20241219_021306-nv4c47le/files/wandb-summary.json @@ -0,0 +1 @@ +{"mean_success_rates":0.45,"SR_test_L5":0.6,"lr":0,"_step":44999,"bc_loss":1.1081001503043808e-06,"epoch":2999,"SR_test_L3":0.6333333333333333,"mean_durations":4.84709279127419,"global_step":44999,"train_action_mse_error":1.8008979168371297e-05,"std_durations":0.003142567013560075,"_timestamp":1.7345537128343365e+09,"train_loss":1.2438786446485514e-06,"test_mean_score":0.45,"_runtime":8126.141499735,"mean_traj_rewards":13042.742712645655,"_wandb":{"runtime":8126}} \ No newline at end of file diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/checkpoints/info.txt b/metaworld_button-press-topdown-dp3-0322_seed0/checkpoints/info.txt new file mode 100644 index 0000000000000000000000000000000000000000..9390f403b52ec26448c5ab32c30ce2c2e41b2eb6 --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/checkpoints/info.txt @@ -0,0 +1 @@ +epoch=0200-test_mean_score=1.000.ckpt \ No newline at end of file diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/config.yaml b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4ef6d34a4d240298664effc48921819b72be019 --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/config.yaml @@ -0,0 +1,242 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: metaworld_button-press-topdown-dp3-0322 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: metaworld_button-press-topdown-dp3-0322 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.18/02.29.37_train_dp3_button-press-topdown + wandb_name_base: 2024.12.18-02.29.37_train_dp3_button-press-topdown +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-topdown_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + device: cuda:0 + eval_episodes: 20 + fps: 10 + n_action_steps: 3 + n_envs: null + n_obs_steps: 2 + n_test: null + n_train: null + task_name: button-press-topdown + use_point_crop: true + name: button-press-topdown + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: train_dp3 +task_name: + value: button-press-topdown +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/requirements.txt b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4932de6b67b7954a352c7097d35162e7d60ed6c9 --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/requirements.txt @@ -0,0 +1,175 @@ +diffusion_policy_3d==0.0.0 +certifi==2024.12.14 +PyYAML==6.0.2 +dash-table==5.0.0 +dm_control==1.0.23 +einops==0.4.1 +ptyprocess==0.7.0 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +nbformat==5.10.4 +decorator==4.4.2 +importlib_metadata==8.5.0 +numcodecs==0.12.1 +requests==2.32.3 +ipywidgets==8.1.5 +urllib3==2.2.3 +platformdirs==4.3.6 +imageio-ffmpeg==0.5.1 +dash==2.18.2 +Cython==0.29.35 +mujoco-py==2.1.2.14 +mpmath==1.3.0 +typing_extensions==4.12.2 +dm-tree==0.1.8 +imageio==2.35.1 +dill==0.3.5.1 +joblib==1.4.2 +attrs==24.3.0 +torchvision==0.19.1+cu121 +kiwisolver==1.4.7 +numpy==1.23.5 +pillow==10.2.0 +nvidia-cuda-cupti-cu12==12.1.105 +pip==24.2 +llvmlite==0.39.1 +tenacity==9.0.0 +scipy==1.10.1 +charset-normalizer==3.4.0 +matplotlib==3.7.5 +jupyterlab_widgets==3.0.13 +patchelf==0.17.2.0 +wcwidth==0.2.13 +pexpect==4.9.0 +plotly==5.24.1 +ipdb==0.13.13 +referencing==0.35.1 +mujoco==3.2.3 +portalocker==3.0.0 +jedi==0.19.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +nvidia-nvtx-cu12==12.1.105 +python-dateutil==2.9.0.post0 +packaging==24.2 +click==8.1.7 +pycparser==2.22 +blinker==1.8.2 +sympy==1.13.1 +Werkzeug==3.0.6 +six==1.17.0 +stack-data==0.6.3 +gpustat==1.1.1 +threadpoolctl==3.5.0 +traitlets==5.14.3 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.1.105 +zarr==2.12.0 +pandas==2.0.3 +wheel==0.44.0 +nvidia-nccl-cu12==2.20.5 +comm==0.2.2 +proglog==0.1.10 +parso==0.8.4 +open3d==0.18.0 +nvidia-cusparse-cu12==12.1.0.106 +antlr4-python3-runtime==4.9.3 +fvcore==0.1.5.post20221221 +natsort==8.4.0 +cloudpickle==3.1.0 +ConfigArgParse==1.7 +fonttools==4.55.3 +etils==1.3.0 +torchaudio==2.4.1+cu121 +pickleshare==0.7.5 +eval_type_backport==0.2.0 +executing==2.1.0 +addict==2.4.0 +protobuf==5.29.1 +setproctitle==1.3.4 +Flask==3.0.3 +filelock==3.13.1 +regex==2024.11.6 +PyOpenGL==3.1.7 +nvidia-curand-cu12==10.3.2.106 +matplotlib-inline==0.1.7 +transforms3d==0.4.2 +pydantic==2.10.3 +cffi==1.17.1 +rpds-py==0.20.1 +diffusers==0.11.1 +docker-pycreds==0.4.0 +pyparsing==3.1.4 +absl-py==2.1.0 +nvidia-cufft-cu12==11.0.2.54 +contourpy==1.1.1 +fastjsonschema==2.21.1 +pkgutil_resolve_name==1.3.10 +dash-html-components==2.0.0 +asciitree==0.3.3 +fasteners==0.19 +glfw==2.8.0 +omegaconf==2.3.0 +asttokens==3.0.0 +moviepy==1.0.3 +GitPython==3.1.43 +opencv-python==4.10.0.84 +zipp==3.20.2 +tabulate==0.9.0 +iopath==0.1.10 +labmaze==1.0.6 +numba==0.56.4 +nvidia-cuda-runtime-cu12==12.1.105 +dm-env==1.6 +cycler==0.12.1 +dash-core-components==2.0.0 +networkx==3.0 +Pygments==2.18.0 +retrying==1.3.4 +jsonschema-specifications==2023.12.1 +setuptools==59.5.0 +av==12.3.0 +itsdangerous==2.2.0 +wandb==0.19.1 +nest-asyncio==1.6.0 +triton==3.0.0 +nvidia-ml-py==12.560.30 +nvidia-cuda-nvrtc-cu12==12.1.105 +tomli==2.2.1 +tqdm==4.67.1 +termcolor==2.4.0 +hydra-core==1.2.0 +smmap==5.0.1 +ipython==8.12.3 +pyquaternion==0.9.9 +idna==3.10 +yacs==0.1.8 +tzdata==2024.2 +torch==2.4.1+cu121 +widgetsnbextension==4.0.13 +scikit-learn==1.3.2 +sapien==2.2.1 +nvidia-cublas-cu12==12.1.3.1 +huggingface-hub==0.21.0 +gitdb==4.0.11 +importlib_resources==6.4.5 +pydantic_core==2.27.1 +jupyter_core==5.7.2 +sentry-sdk==2.19.2 +nvidia-cudnn-cu12==9.1.0.70 +fsspec==2024.2.0 +psutil==6.1.0 +pytz==2024.2 +backcall==0.2.0 +nvidia-cusolver-cu12==11.4.5.107 +lxml==5.3.0 +blessed==1.20.0 +pure_eval==0.2.3 +diffusion_policy_3d==0.0.0 +dexart==0.1.0 +gym==0.21.0 +metaworld==0.1.0 +mj-envs==1.0.0 +mjrl==1.0.0 +pytorch3d==0.7.6 +diffusion_policy_3d==0.0.0 diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-metadata.json b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..811180e7a70712b00a5838aa5765810a746f6ebf --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.17", + "python": "CPython 3.8.20", + "startedAt": "2024-12-17T18:29:40.205080Z", + "args": [ + "--config-name=dp3.yaml", + "task=metaworld_button-press-topdown", + "hydra.run.dir=data/outputs/metaworld_button-press-topdown-dp3-0322_seed0", + "training.debug=False", + "training.seed=0", + "training.device=cuda:0", + "exp_name=metaworld_button-press-topdown-dp3-0322", + "logging.mode=online", + "checkpoint.save_ckpt=True" + ], + "program": "train.py", + "codePath": "3D-Diffusion-Policy/train.py", + "git": { + "remote": "git@github.com:YanjieZe/3D-Diffusion-Policy.git", + "commit": "3153fb32c17308fc828015156480b77f6b2e666a" + }, + "email": "chenhaojun001@gmail.com", + "root": "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0", + "host": "Server-3090", + "executable": "/home/chenhaojun/miniconda3/envs/dp3/bin/python", + "codePathLocal": "train.py", + "cpu_count": 128, + "cpu_count_logical": 256, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "943412031488", + "used": "868376309760" + } + }, + "memory": { + "total": "1081980669952" + }, + "cpu": { + "count": 128, + "countLogical": 256 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.5" +} \ No newline at end of file diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-summary.json b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3ae2d2810249b88851b33db4c3962908f9739c3d --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/files/wandb-summary.json @@ -0,0 +1 @@ +{"global_step":44999,"SR_test_L5":1,"_runtime":7859.566856332,"train_loss":5.629131862860959e-07,"test_mean_score":1,"mean_traj_rewards":29227.664171579825,"lr":0,"train_action_mse_error":1.17400359158637e-05,"epoch":2999,"_step":44999,"SR_test_L3":1,"bc_loss":6.396743970071839e-07,"_wandb":{"runtime":7859},"_timestamp":1.7344680397637768e+09,"mean_success_rates":1} \ No newline at end of file diff --git a/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..63eafbb19c408396f96acb45c1feeb9b930353cf --- /dev/null +++ b/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log @@ -0,0 +1,24 @@ +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_setup.py:_flush():68] Configure stats pid to 2040078 +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug.log +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0/wandb/run-20241218_022940-yb3y9h3p/logs/debug-internal.log +2024-12-18 02:29:40,171 INFO MainThread:2040078 [wandb_init.py:init():644] calling init triggers +2024-12-18 02:29:40,172 INFO MainThread:2040078 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'button-press-topdown', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'button-press-topdown', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_button-press-topdown_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'button-press-topdown', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_button-press-topdown-dp3-0322', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_button-press-topdown-dp3-0322', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/02.29.37_train_dp3_button-press-topdown', 'wandb_name_base': '2024.12.18-02.29.37_train_dp3_button-press-topdown'}} +2024-12-18 02:29:40,172 INFO MainThread:2040078 [wandb_init.py:init():680] starting backend +2024-12-18 02:29:40,172 INFO MainThread:2040078 [wandb_init.py:init():684] sending inform_init request +2024-12-18 02:29:40,204 INFO MainThread:2040078 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 02:29:40,204 INFO MainThread:2040078 [wandb_init.py:init():697] backend started and connected +2024-12-18 02:29:40,209 INFO MainThread:2040078 [wandb_init.py:init():790] updated telemetry +2024-12-18 02:29:40,368 INFO MainThread:2040078 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 02:29:41,789 INFO MainThread:2040078 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 02:29:41,910 INFO MainThread:2040078 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 02:29:41,910 INFO MainThread:2040078 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 02:29:41,911 INFO MainThread:2040078 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 02:29:41,911 INFO MainThread:2040078 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 02:29:41,913 INFO MainThread:2040078 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 02:29:41,913 INFO MainThread:2040078 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-dp3-0322_seed0'} +2024-12-18 04:40:39,772 WARNING MsgRouterThr:2040078 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/config.yaml b/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bee7e932d6b8fb30996b2272d39d70ab8c7b72ef --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/config.yaml @@ -0,0 +1,156 @@ +task: + name: button-press-topdown-wall + task_name: ${name} + shape_meta: + obs: + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + agent_pos: + shape: + - 9 + type: low_dim + action: + shape: + - 4 + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + eval_episodes: 20 + n_obs_steps: ${n_obs_steps} + n_action_steps: ${n_action_steps} + fps: 10 + n_envs: null + n_train: null + n_test: null + task_name: ${task_name} + device: ${training.device} + use_point_crop: ${policy.use_point_crop} + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-topdown-wall_expert.zarr + horizon: ${horizon} + pad_before: ${eval:'${n_obs_steps}-1'} + pad_after: ${eval:'${n_action_steps}-1'} + seed: 42 + val_ratio: 0.02 + max_train_episodes: 90 +name: train_dp3 +task_name: ${task.name} +shape_meta: ${task.shape_meta} +exp_name: metaworld_button-press-topdown-wall-dp3-1218 +horizon: 4 +n_obs_steps: 2 +n_action_steps: 3 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +keypoint_visible_rate: 1.0 +obs_as_global_cond: true +policy: + _target_: diffusion_policy_3d.policy.dp3.DP3 + use_point_crop: true + condition_type: film + use_down_condition: true + use_mid_condition: true + use_up_condition: true + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + crop_shape: + - 80 + - 80 + encoder_output_dim: 64 + horizon: ${horizon} + kernel_size: 5 + n_action_steps: ${n_action_steps} + n_groups: 8 + n_obs_steps: ${n_obs_steps} + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + clip_sample: true + set_alpha_to_one: true + steps_offset: 0 + prediction_type: sample + num_inference_steps: 10 + obs_as_global_cond: true + shape_meta: ${shape_meta} + use_pc_color: false + pointnet_type: pointnet + pointcloud_encoder_cfg: + in_channels: 3 + out_channels: ${policy.encoder_output_dim} + use_layernorm: true + final_norm: layernorm + normal_channel: false +ema: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 +dataloader: + batch_size: 128 + num_workers: 8 + shuffle: true + pin_memory: true + persistent_workers: false +val_dataloader: + batch_size: 128 + num_workers: 8 + shuffle: false + pin_memory: true + persistent_workers: false +optimizer: + _target_: torch.optim.AdamW + lr: 0.0001 + betas: + - 0.95 + - 0.999 + eps: 1.0e-08 + weight_decay: 1.0e-06 +training: + device: cuda:0 + seed: 0 + debug: false + resume: true + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 3000 + gradient_accumulate_every: 1 + use_ema: true + rollout_every: 200 + checkpoint_every: 200 + val_every: 1 + sample_every: 5 + max_train_steps: null + max_val_steps: null + tqdm_interval_sec: 1.0 +logging: + group: ${exp_name} + id: null + mode: '' + name: ${training.seed} + project: dp3 + resume: true + tags: + - dp3 +checkpoint: + save_ckpt: '' + topk: + monitor_key: test_mean_score + mode: max + k: 1 + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + save_last_ckpt: true + save_last_snapshot: false +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/overrides.yaml b/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d398806b7bbefaf9332a8e8463855ca5c2407211 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/.hydra/overrides.yaml @@ -0,0 +1,7 @@ +- task=metaworld_button-press-topdown-wall +- training.debug=False +- training.seed=0 +- training.device=cuda:0 +- exp_name=metaworld_button-press-topdown-wall-dp3-1218 +- logging.mode= +- checkpoint.save_ckpt= diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/eval_results.json b/metaworld_button-press-topdown-wall-dp3-1218_seed0/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..e43201fe6f471ab3604350f9f6ed472919ef5944 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/eval_results.json @@ -0,0 +1,9 @@ +{ + "mean_traj_rewards": 262.50912456420383, + "mean_success_rates": 1.0, + "mean_durations": 4.872495170589536, + "std_durations": 0.07859229908783016, + "test_mean_score": 1.0, + "SR_test_L3": 1.0, + "SR_test_L5": 1.0 +} \ No newline at end of file diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/files/wandb-summary.json b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..501916b9b3b9cb78400ebd38bce0d67f6d125895 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/files/wandb-summary.json @@ -0,0 +1 @@ +{"bc_loss":9.746736395754851e-07,"_timestamp":1.7345114860345306e+09,"global_step":44999,"train_loss":6.30829500399462e-07,"lr":0,"_wandb":{"runtime":7898},"mean_success_rates":1,"_runtime":7898.281424519,"test_mean_score":1,"train_action_mse_error":1.1380741852917708e-05,"SR_test_L3":1,"SR_test_L5":1,"epoch":2999,"mean_traj_rewards":201.4324005806066,"_step":44999} \ No newline at end of file diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..07db55eadb5083757caebc9d9446dc949e2b53c2 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log @@ -0,0 +1,24 @@ +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_setup.py:_flush():68] Configure stats pid to 2434667 +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug.log +2024-12-18 14:33:07,732 INFO MainThread:2434667 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_143307-z8y9ggjn/logs/debug-internal.log +2024-12-18 14:33:07,733 INFO MainThread:2434667 [wandb_init.py:init():644] calling init triggers +2024-12-18 14:33:07,733 INFO MainThread:2434667 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'button-press-topdown-wall', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'button-press-topdown-wall', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_button-press-topdown-wall_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'button-press-topdown-wall', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_button-press-topdown-wall-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_button-press-topdown-wall-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/14.33.05_train_dp3_button-press-topdown-wall', 'wandb_name_base': '2024.12.18-14.33.05_train_dp3_button-press-topdown-wall'}} +2024-12-18 14:33:07,733 INFO MainThread:2434667 [wandb_init.py:init():680] starting backend +2024-12-18 14:33:07,733 INFO MainThread:2434667 [wandb_init.py:init():684] sending inform_init request +2024-12-18 14:33:07,760 INFO MainThread:2434667 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 14:33:07,761 INFO MainThread:2434667 [wandb_init.py:init():697] backend started and connected +2024-12-18 14:33:07,766 INFO MainThread:2434667 [wandb_init.py:init():790] updated telemetry +2024-12-18 14:33:07,921 INFO MainThread:2434667 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 14:33:09,409 INFO MainThread:2434667 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 14:33:09,542 INFO MainThread:2434667 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 14:33:09,542 INFO MainThread:2434667 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 14:33:09,543 INFO MainThread:2434667 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 14:33:09,543 INFO MainThread:2434667 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 14:33:09,545 INFO MainThread:2434667 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 14:33:09,546 INFO MainThread:2434667 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-topdown-wall-dp3-1218_seed0'} +2024-12-18 16:44:46,043 WARNING MsgRouterThr:2434667 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_175628-z8y9ggjn/files/output.log b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_175628-z8y9ggjn/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b7e7b605132a3b20b157bbf44fe8f2f8afb8dbf9 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/run-20241218_175628-z8y9ggjn/files/output.log @@ -0,0 +1,131 @@ +Eval in Metaworld button-press-topdown-wall Pointcloud Env: 0%| | 0/20 [00:00 + main() + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/main.py", line 90, in decorated_main + _run_hydra( + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 389, in _run_hydra + _run_app( + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 452, in _run_app + run_and_report( + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 213, in run_and_report + return func() + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/utils.py", line 453, in + lambda: hydra.run( + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/_internal/hydra.py", line 119, in run + ret = run_job( + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/hydra/core/utils.py", line 186, in run_job + ret.return_value = task_function(task_cfg) + File "train.py", line 503, in main + workspace.run() + File "train.py", line 186, in run + for batch_idx, batch in enumerate(tepoch): + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/tqdm/std.py", line 1181, in __iter__ + for obj in iterable: + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 630, in __next__ + data = self._next_data() + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1316, in _next_data + self._shutdown_workers() + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 1441, in _shutdown_workers + w.join(timeout=_utils.MP_STATUS_CHECK_INTERVAL) + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/process.py", line 149, in join + res = self._popen.wait(timeout) + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/popen_fork.py", line 44, in wait + if not wait([self.sentinel], timeout): + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/multiprocessing/connection.py", line 931, in wait + ready = selector.select(timeout) + File "/home/chenhaojun/miniconda3/envs/dp3/lib/python3.8/selectors.py", line 415, in select + fd_event_list = self._selector.poll(timeout) +KeyboardInterrupt diff --git a/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/wandb-resume.json b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/wandb-resume.json new file mode 100644 index 0000000000000000000000000000000000000000..5590a837d72760a3bd5b72c3cf38a19ce3a352c6 --- /dev/null +++ b/metaworld_button-press-topdown-wall-dp3-1218_seed0/wandb/wandb-resume.json @@ -0,0 +1 @@ +{"run_id": "z8y9ggjn"} \ No newline at end of file diff --git a/metaworld_button-press-wall-dp3-1218_seed0/.hydra/config.yaml b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d10ccb856fc28f31f2fc56a8365960982874a479 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/config.yaml @@ -0,0 +1,156 @@ +task: + name: button-press-wall + task_name: ${name} + shape_meta: + obs: + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + agent_pos: + shape: + - 9 + type: low_dim + action: + shape: + - 4 + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + eval_episodes: 20 + n_obs_steps: ${n_obs_steps} + n_action_steps: ${n_action_steps} + fps: 10 + n_envs: null + n_train: null + n_test: null + task_name: ${task_name} + device: ${training.device} + use_point_crop: ${policy.use_point_crop} + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-wall_expert.zarr + horizon: ${horizon} + pad_before: ${eval:'${n_obs_steps}-1'} + pad_after: ${eval:'${n_action_steps}-1'} + seed: 42 + val_ratio: 0.02 + max_train_episodes: 90 +name: train_dp3 +task_name: ${task.name} +shape_meta: ${task.shape_meta} +exp_name: metaworld_button-press-wall-dp3-1218 +horizon: 4 +n_obs_steps: 2 +n_action_steps: 3 +n_latency_steps: 0 +dataset_obs_steps: ${n_obs_steps} +keypoint_visible_rate: 1.0 +obs_as_global_cond: true +policy: + _target_: diffusion_policy_3d.policy.dp3.DP3 + use_point_crop: true + condition_type: film + use_down_condition: true + use_mid_condition: true + use_up_condition: true + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + crop_shape: + - 80 + - 80 + encoder_output_dim: 64 + horizon: ${horizon} + kernel_size: 5 + n_action_steps: ${n_action_steps} + n_groups: 8 + n_obs_steps: ${n_obs_steps} + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + num_train_timesteps: 100 + beta_start: 0.0001 + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + clip_sample: true + set_alpha_to_one: true + steps_offset: 0 + prediction_type: sample + num_inference_steps: 10 + obs_as_global_cond: true + shape_meta: ${shape_meta} + use_pc_color: false + pointnet_type: pointnet + pointcloud_encoder_cfg: + in_channels: 3 + out_channels: ${policy.encoder_output_dim} + use_layernorm: true + final_norm: layernorm + normal_channel: false +ema: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + update_after_step: 0 + inv_gamma: 1.0 + power: 0.75 + min_value: 0.0 + max_value: 0.9999 +dataloader: + batch_size: 128 + num_workers: 8 + shuffle: true + pin_memory: true + persistent_workers: false +val_dataloader: + batch_size: 128 + num_workers: 8 + shuffle: false + pin_memory: true + persistent_workers: false +optimizer: + _target_: torch.optim.AdamW + lr: 0.0001 + betas: + - 0.95 + - 0.999 + eps: 1.0e-08 + weight_decay: 1.0e-06 +training: + device: cuda:0 + seed: 0 + debug: false + resume: true + lr_scheduler: cosine + lr_warmup_steps: 500 + num_epochs: 3000 + gradient_accumulate_every: 1 + use_ema: true + rollout_every: 200 + checkpoint_every: 200 + val_every: 1 + sample_every: 5 + max_train_steps: null + max_val_steps: null + tqdm_interval_sec: 1.0 +logging: + group: ${exp_name} + id: null + mode: '' + name: ${training.seed} + project: dp3 + resume: true + tags: + - dp3 +checkpoint: + save_ckpt: '' + topk: + monitor_key: test_mean_score + mode: max + k: 1 + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + save_last_ckpt: true + save_last_snapshot: false +multi_run: + run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name} diff --git a/metaworld_button-press-wall-dp3-1218_seed0/.hydra/hydra.yaml b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2af4055993865a3b74d0c937663177a21c22adad --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: data/outputs/metaworld_button-press-wall-dp3-1218_seed0 + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=data/outputs/metaworld_button-press-wall-dp3-1218_seed0 + - hydra.mode=RUN + task: + - task=metaworld_button-press-wall + - training.debug=False + - training.seed=0 + - training.device=cuda:0 + - exp_name=metaworld_button-press-wall-dp3-1218 + - logging.mode= + - checkpoint.save_ckpt= + job: + name: eval + chdir: null + override_dirname: checkpoint.save_ckpt=,exp_name=metaworld_button-press-wall-dp3-1218,logging.mode=,task=metaworld_button-press-wall,training.debug=False,training.device=cuda:0,training.seed=0 + id: ??? + num: ??? + config_name: dp3.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.2.0 + version_base: '1.2' + cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0 + choices: + task: metaworld_button-press-wall + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/metaworld_button-press-wall-dp3-1218_seed0/.hydra/overrides.yaml b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..338c31239861edf8b16bcde3756307aa713b58ce --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/.hydra/overrides.yaml @@ -0,0 +1,7 @@ +- task=metaworld_button-press-wall +- training.debug=False +- training.seed=0 +- training.device=cuda:0 +- exp_name=metaworld_button-press-wall-dp3-1218 +- logging.mode= +- checkpoint.save_ckpt= diff --git a/metaworld_button-press-wall-dp3-1218_seed0/eval.log b/metaworld_button-press-wall-dp3-1218_seed0/eval.log new file mode 100644 index 0000000000000000000000000000000000000000..0f72dcf3570452c51d23730c6e8069db1cdb9399 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/eval.log @@ -0,0 +1 @@ +[2024-12-18 19:31:00,745][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 diff --git a/metaworld_button-press-wall-dp3-1218_seed0/eval_results.json b/metaworld_button-press-wall-dp3-1218_seed0/eval_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c4e23eab88520938451c47fd97d090e6f3f41b49 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/eval_results.json @@ -0,0 +1,9 @@ +{ + "mean_traj_rewards": 358.85671363099266, + "mean_success_rates": 0.95, + "mean_durations": 4.826406791713088, + "std_durations": 0.07742426010869517, + "test_mean_score": 0.95, + "SR_test_L3": 0.95, + "SR_test_L5": 0.95 +} \ No newline at end of file diff --git a/metaworld_button-press-wall-dp3-1218_seed0/train.log b/metaworld_button-press-wall-dp3-1218_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1561d65c2180957e118fac887c480e6f9c27c337 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/train.log @@ -0,0 +1 @@ +[2024-12-18 17:56:28,485][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 diff --git a/metaworld_button-press-wall-dp3-1218_seed0/wandb/debug.log b/metaworld_button-press-wall-dp3-1218_seed0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..38b2e2a573a88a6871cfe1be9eed656a7829eeb2 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/wandb/debug.log @@ -0,0 +1,24 @@ +2024-12-18 17:56:35,952 INFO MainThread:2877151 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 17:56:35,952 INFO MainThread:2877151 [wandb_setup.py:_flush():68] Configure stats pid to 2877151 +2024-12-18 17:56:35,952 INFO MainThread:2877151 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 17:56:35,952 INFO MainThread:2877151 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 17:56:35,952 INFO MainThread:2877151 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/logs/debug.log +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/logs/debug-internal.log +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:init():644] calling init triggers +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'button-press-wall', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'button-press-wall', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_button-press-wall_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'button-press-wall', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_button-press-wall-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_button-press-wall-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/17.56.33_train_dp3_button-press-wall', 'wandb_name_base': '2024.12.18-17.56.33_train_dp3_button-press-wall'}} +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:init():680] starting backend +2024-12-18 17:56:35,953 INFO MainThread:2877151 [wandb_init.py:init():684] sending inform_init request +2024-12-18 17:56:35,982 INFO MainThread:2877151 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 17:56:35,982 INFO MainThread:2877151 [wandb_init.py:init():697] backend started and connected +2024-12-18 17:56:35,987 INFO MainThread:2877151 [wandb_init.py:init():790] updated telemetry +2024-12-18 17:56:36,148 INFO MainThread:2877151 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 17:56:38,709 INFO MainThread:2877151 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 17:56:38,831 INFO MainThread:2877151 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 17:56:38,831 INFO MainThread:2877151 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 17:56:38,831 INFO MainThread:2877151 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 17:56:38,831 INFO MainThread:2877151 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 17:56:38,834 INFO MainThread:2877151 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 17:56:38,834 INFO MainThread:2877151 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0'} +2024-12-18 17:59:24,401 WARNING MsgRouterThr:2877151 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/config.yaml b/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef8f7fd823da758f9340e8c831c94e1eb2486a5 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/config.yaml @@ -0,0 +1,242 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: metaworld_button-press-wall-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: metaworld_button-press-wall-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.18/17.56.33_train_dp3_button-press-wall + wandb_name_base: 2024.12.18-17.56.33_train_dp3_button-press-wall +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/metaworld_button-press-wall_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + device: cuda:0 + eval_episodes: 20 + fps: 10 + n_action_steps: 3 + n_envs: null + n_obs_steps: 2 + n_test: null + n_train: null + task_name: button-press-wall + use_point_crop: true + name: button-press-wall + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: train_dp3 +task_name: + value: button-press-wall +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/wandb-metadata.json b/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fca3efe24117a41e0f72b4a1b469f52d48d435e9 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/wandb/run-20241218_175635-nciey1ys/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.17", + "python": "CPython 3.8.20", + "startedAt": "2024-12-18T09:56:35.983114Z", + "args": [ + "--config-name=dp3.yaml", + "task=metaworld_button-press-wall", + "hydra.run.dir=data/outputs/metaworld_button-press-wall-dp3-1218_seed0", + "training.debug=False", + "training.seed=0", + "training.device=cuda:0", + "exp_name=metaworld_button-press-wall-dp3-1218", + "logging.mode=online", + "checkpoint.save_ckpt=True" + ], + "program": "train.py", + "codePath": "3D-Diffusion-Policy/train.py", + "git": { + "remote": "git@github.com:YanjieZe/3D-Diffusion-Policy.git", + "commit": "3153fb32c17308fc828015156480b77f6b2e666a" + }, + "email": "chenhaojun001@gmail.com", + "root": "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_button-press-wall-dp3-1218_seed0", + "host": "Server-3090", + "executable": "/home/chenhaojun/miniconda3/envs/dp3/bin/python", + "codePathLocal": "train.py", + "cpu_count": 128, + "cpu_count_logical": 256, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "943412031488", + "used": "860777975808" + } + }, + "memory": { + "total": "1081980669952" + }, + "cpu": { + "count": 128, + "countLogical": 256 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.5" +} \ No newline at end of file diff --git a/metaworld_button-press-wall-dp3-1218_seed0/wandb/wandb-resume.json b/metaworld_button-press-wall-dp3-1218_seed0/wandb/wandb-resume.json new file mode 100644 index 0000000000000000000000000000000000000000..6bd2e5a9a3850480867af8a80265602c3690fec5 --- /dev/null +++ b/metaworld_button-press-wall-dp3-1218_seed0/wandb/wandb-resume.json @@ -0,0 +1 @@ +{"run_id": "nciey1ys"} \ No newline at end of file diff --git a/metaworld_coffee-button-dp3-0322_seed0/.hydra/hydra.yaml b/metaworld_coffee-button-dp3-0322_seed0/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4edaff9e1d62ddc8306ebafd52eaffa77f6e602f --- /dev/null +++ b/metaworld_coffee-button-dp3-0322_seed0/.hydra/hydra.yaml @@ -0,0 +1,163 @@ +hydra: + run: + dir: data/outputs/metaworld_coffee-button-dp3-0322_seed0 + sweep: + dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.run.dir=data/outputs/metaworld_coffee-button-dp3-0322_seed0 + - hydra.mode=RUN + task: + - task=metaworld_coffee-button + - training.debug=False + - training.seed=0 + - training.device=cuda:0 + - exp_name=metaworld_coffee-button-dp3-0322 + - logging.mode=online + - checkpoint.save_ckpt=True + job: + name: train + chdir: null + override_dirname: checkpoint.save_ckpt=True,exp_name=metaworld_coffee-button-dp3-0322,logging.mode=online,task=metaworld_coffee-button,training.debug=False,training.device=cuda:0,training.seed=0 + id: ??? + num: ??? + config_name: dp3.yaml + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.2.0 + version_base: '1.2' + cwd: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/diffusion_policy_3d/config + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-button-dp3-0322_seed0 + choices: + task: metaworld_coffee-button + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: basic + hydra/output: default + verbose: false diff --git a/metaworld_coffee-button-dp3-0322_seed0/checkpoints/info.txt b/metaworld_coffee-button-dp3-0322_seed0/checkpoints/info.txt new file mode 100644 index 0000000000000000000000000000000000000000..9390f403b52ec26448c5ab32c30ce2c2e41b2eb6 --- /dev/null +++ b/metaworld_coffee-button-dp3-0322_seed0/checkpoints/info.txt @@ -0,0 +1 @@ +epoch=0200-test_mean_score=1.000.ckpt \ No newline at end of file diff --git a/metaworld_coffee-button-dp3-0322_seed0/wandb/debug-internal.log b/metaworld_coffee-button-dp3-0322_seed0/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..84edf49b1582976c0fc346654ebf659ce504fe2f --- /dev/null +++ b/metaworld_coffee-button-dp3-0322_seed0/wandb/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-12-18T04:41:02.045606381+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-18T04:41:02.045634041+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-button-dp3-0322_seed0/wandb/run-20241218_044102-v4stgwmg/logs/debug-core.log"} +{"time":"2024-12-18T04:41:02.367222173+08:00","level":"INFO","msg":"created new stream","id":"v4stgwmg"} +{"time":"2024-12-18T04:41:02.367271313+08:00","level":"INFO","msg":"stream: started","id":"v4stgwmg"} +{"time":"2024-12-18T04:41:02.367315902+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"v4stgwmg"} +{"time":"2024-12-18T04:41:02.367599701+08:00","level":"INFO","msg":"sender: started","stream_id":"v4stgwmg"} +{"time":"2024-12-18T04:41:02.367644801+08:00","level":"INFO","msg":"handler: started","stream_id":"v4stgwmg"} +{"time":"2024-12-18T04:41:04.96653204+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-18T06:54:04.048684922+08:00","level":"INFO","msg":"stream: closing","id":"v4stgwmg"} +{"time":"2024-12-18T06:54:04.048730221+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-18T06:54:04.049959439+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-18T06:54:07.334785176+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-18T06:54:09.49861508+08:00","level":"INFO","msg":"handler: closed","stream_id":"v4stgwmg"} +{"time":"2024-12-18T06:54:09.4986585+08:00","level":"INFO","msg":"sender: closed","stream_id":"v4stgwmg"} +{"time":"2024-12-18T06:54:09.49864263+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"v4stgwmg"} +{"time":"2024-12-18T06:54:09.498759969+08:00","level":"INFO","msg":"stream: closed","id":"v4stgwmg"} diff --git a/metaworld_coffee-button-dp3-0322_seed0/wandb/wandb-resume.json b/metaworld_coffee-button-dp3-0322_seed0/wandb/wandb-resume.json new file mode 100644 index 0000000000000000000000000000000000000000..87fa3d44ef3dfdda3ebb455598725dc413392298 --- /dev/null +++ b/metaworld_coffee-button-dp3-0322_seed0/wandb/wandb-resume.json @@ -0,0 +1 @@ +{"run_id": "v4stgwmg"} \ No newline at end of file diff --git a/metaworld_coffee-push-dp3-1218_seed0/train.log b/metaworld_coffee-push-dp3-1218_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1133476d4bb2f27040df08f930b9d8fe374ef672 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/train.log @@ -0,0 +1,2 @@ +[2024-12-18 20:53:00,891][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 +[2024-12-18 20:53:21,556][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/debug.log b/metaworld_coffee-push-dp3-1218_seed0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f8067b04c0afc8da520b50b10dac2f1b9a62f08d --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/debug.log @@ -0,0 +1,25 @@ +2024-12-18 20:53:28,982 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Configure stats pid to 2966139 +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug.log +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-internal.log +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:init():644] calling init triggers +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'coffee-push', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'coffee-push', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_coffee-push_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'coffee-push', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_coffee-push-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_coffee-push-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/20.53.26_train_dp3_coffee-push', 'wandb_name_base': '2024.12.18-20.53.26_train_dp3_coffee-push'}} +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():680] starting backend +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():684] sending inform_init request +2024-12-18 20:53:29,015 INFO MainThread:2966139 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 20:53:29,015 INFO MainThread:2966139 [wandb_init.py:init():697] backend started and connected +2024-12-18 20:53:29,022 INFO MainThread:2966139 [wandb_init.py:init():790] updated telemetry +2024-12-18 20:53:29,172 INFO MainThread:2966139 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 20:53:30,726 INFO MainThread:2966139 [wandb_init.py:init():867] run resumed +2024-12-18 20:53:30,731 INFO MainThread:2966139 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 20:53:30,858 INFO MainThread:2966139 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 20:53:30,858 INFO MainThread:2966139 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 20:53:30,859 INFO MainThread:2966139 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 20:53:30,859 INFO MainThread:2966139 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 20:53:30,860 INFO MainThread:2966139 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 20:53:30,861 INFO MainThread:2966139 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0'} +2024-12-19 00:16:29,480 WARNING MsgRouterThr:2966139 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/files/config.yaml b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e871a227d4033ec76224b49c94bd1260ec3a420f --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/files/config.yaml @@ -0,0 +1,242 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: metaworld_coffee-push-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: metaworld_coffee-push-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.18/20.53.06_train_dp3_coffee-push + wandb_name_base: 2024.12.18-20.53.06_train_dp3_coffee-push +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/metaworld_coffee-push_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + device: cuda:0 + eval_episodes: 20 + fps: 10 + n_action_steps: 3 + n_envs: null + n_obs_steps: 2 + n_test: null + n_train: null + task_name: coffee-push + use_point_crop: true + name: coffee-push + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: train_dp3 +task_name: + value: coffee-push +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/logs/debug-internal.log b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2587cec2b86e267ab70104abf6e6852cb9310a47 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/logs/debug-internal.log @@ -0,0 +1,16 @@ +{"time":"2024-12-18T20:53:09.261065991+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-18T20:53:09.261092471+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205309-ss7puy3o/logs/debug-core.log"} +{"time":"2024-12-18T20:53:09.480346615+08:00","level":"INFO","msg":"created new stream","id":"ss7puy3o"} +{"time":"2024-12-18T20:53:09.480400825+08:00","level":"INFO","msg":"stream: started","id":"ss7puy3o"} +{"time":"2024-12-18T20:53:09.480487594+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:09.480531544+08:00","level":"INFO","msg":"sender: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:09.480619414+08:00","level":"INFO","msg":"handler: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:12.018382277+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-19T00:17:44.116111485+08:00","level":"INFO","msg":"stream: closing","id":"ss7puy3o"} +{"time":"2024-12-19T00:17:44.116160285+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-19T00:17:44.117549337+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-19T00:17:46.890867478+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-19T00:17:48.849035287+08:00","level":"INFO","msg":"handler: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:17:48.849078466+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:17:48.849142776+08:00","level":"INFO","msg":"sender: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:17:48.849255415+08:00","level":"INFO","msg":"stream: closed","id":"ss7puy3o"} diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/config.yaml b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1a78564c9fc8e476de7c23d6b98187dc61497516 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/config.yaml @@ -0,0 +1,244 @@ +_wandb: + value: + cli_version: 0.19.1 + m: [] + python_version: 3.8.20 + t: + "1": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "2": + - 1 + - 5 + - 41 + - 49 + - 50 + - 53 + - 55 + - 83 + "3": + - 5 + - 13 + - 15 + - 16 + - 23 + - 55 + - 61 + - 62 + "4": 3.8.20 + "5": 0.19.1 + "8": + - 5 + "12": 0.19.1 + "13": linux-x86_64 +checkpoint: + value: + save_ckpt: true + save_last_ckpt: true + save_last_snapshot: false + topk: + format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt + k: 1 + mode: max + monitor_key: test_mean_score +dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: true +dataset_obs_steps: + value: 2 +ema: + value: + _target_: diffusion_policy_3d.model.diffusion.ema_model.EMAModel + inv_gamma: 1 + max_value: 0.9999 + min_value: 0 + power: 0.75 + update_after_step: 0 +exp_name: + value: metaworld_coffee-push-dp3-1218 +horizon: + value: 4 +keypoint_visible_rate: + value: 1 +logging: + value: + group: metaworld_coffee-push-dp3-1218 + id: null + mode: online + name: "0" + project: dp3 + resume: true + tags: + - dp3 +multi_run: + value: + run_dir: data/outputs/2024.12.18/20.53.26_train_dp3_coffee-push + wandb_name_base: 2024.12.18-20.53.26_train_dp3_coffee-push +n_action_steps: + value: 3 +n_latency_steps: + value: 0 +n_obs_steps: + value: 2 +name: + value: train_dp3 +obs_as_global_cond: + value: true +optimizer: + value: + _target_: torch.optim.AdamW + betas: + - 0.95 + - 0.999 + eps: 1e-08 + lr: 0.0001 + weight_decay: 1e-06 +output_dir: + value: /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0 +policy: + value: + _target_: diffusion_policy_3d.policy.dp3.DP3 + condition_type: film + crop_shape: + - 80 + - 80 + diffusion_step_embed_dim: 128 + down_dims: + - 512 + - 1024 + - 2048 + encoder_output_dim: 64 + horizon: 4 + kernel_size: 5 + n_action_steps: 3 + n_groups: 8 + n_obs_steps: 2 + noise_scheduler: + _target_: diffusers.schedulers.scheduling_ddim.DDIMScheduler + beta_end: 0.02 + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + clip_sample: true + num_train_timesteps: 100 + prediction_type: sample + set_alpha_to_one: true + steps_offset: 0 + num_inference_steps: 10 + obs_as_global_cond: true + pointcloud_encoder_cfg: + final_norm: layernorm + in_channels: 3 + normal_channel: false + out_channels: 64 + use_layernorm: true + pointnet_type: pointnet + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + use_down_condition: true + use_mid_condition: true + use_pc_color: false + use_point_crop: true + use_up_condition: true +shape_meta: + value: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud +task: + value: + dataset: + _target_: diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset + horizon: 4 + max_train_episodes: 90 + pad_after: 2 + pad_before: 1 + seed: 42 + val_ratio: 0.02 + zarr_path: /data/haojun/datasets/3d-dp/metaworld_coffee-push_expert.zarr + env_runner: + _target_: diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner + device: cuda:0 + eval_episodes: 20 + fps: 10 + n_action_steps: 3 + n_envs: null + n_obs_steps: 2 + n_test: null + n_train: null + task_name: coffee-push + use_point_crop: true + name: coffee-push + shape_meta: + action: + shape: + - 4 + obs: + agent_pos: + shape: + - 9 + type: low_dim + point_cloud: + shape: + - 512 + - 3 + type: point_cloud + task_name: train_dp3 +task_name: + value: coffee-push +training: + value: + checkpoint_every: 200 + debug: false + device: cuda:0 + gradient_accumulate_every: 1 + lr_scheduler: cosine + lr_warmup_steps: 500 + max_train_steps: null + max_val_steps: null + num_epochs: 3000 + resume: true + rollout_every: 200 + sample_every: 5 + seed: 0 + tqdm_interval_sec: 1 + use_ema: true + val_every: 1 +val_dataloader: + value: + batch_size: 128 + num_workers: 8 + persistent_workers: false + pin_memory: true + shuffle: false diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/requirements.txt b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4932de6b67b7954a352c7097d35162e7d60ed6c9 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/requirements.txt @@ -0,0 +1,175 @@ +diffusion_policy_3d==0.0.0 +certifi==2024.12.14 +PyYAML==6.0.2 +dash-table==5.0.0 +dm_control==1.0.23 +einops==0.4.1 +ptyprocess==0.7.0 +jsonschema==4.23.0 +prompt_toolkit==3.0.48 +nbformat==5.10.4 +decorator==4.4.2 +importlib_metadata==8.5.0 +numcodecs==0.12.1 +requests==2.32.3 +ipywidgets==8.1.5 +urllib3==2.2.3 +platformdirs==4.3.6 +imageio-ffmpeg==0.5.1 +dash==2.18.2 +Cython==0.29.35 +mujoco-py==2.1.2.14 +mpmath==1.3.0 +typing_extensions==4.12.2 +dm-tree==0.1.8 +imageio==2.35.1 +dill==0.3.5.1 +joblib==1.4.2 +attrs==24.3.0 +torchvision==0.19.1+cu121 +kiwisolver==1.4.7 +numpy==1.23.5 +pillow==10.2.0 +nvidia-cuda-cupti-cu12==12.1.105 +pip==24.2 +llvmlite==0.39.1 +tenacity==9.0.0 +scipy==1.10.1 +charset-normalizer==3.4.0 +matplotlib==3.7.5 +jupyterlab_widgets==3.0.13 +patchelf==0.17.2.0 +wcwidth==0.2.13 +pexpect==4.9.0 +plotly==5.24.1 +ipdb==0.13.13 +referencing==0.35.1 +mujoco==3.2.3 +portalocker==3.0.0 +jedi==0.19.2 +Jinja2==3.1.3 +MarkupSafe==2.1.5 +nvidia-nvtx-cu12==12.1.105 +python-dateutil==2.9.0.post0 +packaging==24.2 +click==8.1.7 +pycparser==2.22 +blinker==1.8.2 +sympy==1.13.1 +Werkzeug==3.0.6 +six==1.17.0 +stack-data==0.6.3 +gpustat==1.1.1 +threadpoolctl==3.5.0 +traitlets==5.14.3 +annotated-types==0.7.0 +nvidia-nvjitlink-cu12==12.1.105 +zarr==2.12.0 +pandas==2.0.3 +wheel==0.44.0 +nvidia-nccl-cu12==2.20.5 +comm==0.2.2 +proglog==0.1.10 +parso==0.8.4 +open3d==0.18.0 +nvidia-cusparse-cu12==12.1.0.106 +antlr4-python3-runtime==4.9.3 +fvcore==0.1.5.post20221221 +natsort==8.4.0 +cloudpickle==3.1.0 +ConfigArgParse==1.7 +fonttools==4.55.3 +etils==1.3.0 +torchaudio==2.4.1+cu121 +pickleshare==0.7.5 +eval_type_backport==0.2.0 +executing==2.1.0 +addict==2.4.0 +protobuf==5.29.1 +setproctitle==1.3.4 +Flask==3.0.3 +filelock==3.13.1 +regex==2024.11.6 +PyOpenGL==3.1.7 +nvidia-curand-cu12==10.3.2.106 +matplotlib-inline==0.1.7 +transforms3d==0.4.2 +pydantic==2.10.3 +cffi==1.17.1 +rpds-py==0.20.1 +diffusers==0.11.1 +docker-pycreds==0.4.0 +pyparsing==3.1.4 +absl-py==2.1.0 +nvidia-cufft-cu12==11.0.2.54 +contourpy==1.1.1 +fastjsonschema==2.21.1 +pkgutil_resolve_name==1.3.10 +dash-html-components==2.0.0 +asciitree==0.3.3 +fasteners==0.19 +glfw==2.8.0 +omegaconf==2.3.0 +asttokens==3.0.0 +moviepy==1.0.3 +GitPython==3.1.43 +opencv-python==4.10.0.84 +zipp==3.20.2 +tabulate==0.9.0 +iopath==0.1.10 +labmaze==1.0.6 +numba==0.56.4 +nvidia-cuda-runtime-cu12==12.1.105 +dm-env==1.6 +cycler==0.12.1 +dash-core-components==2.0.0 +networkx==3.0 +Pygments==2.18.0 +retrying==1.3.4 +jsonschema-specifications==2023.12.1 +setuptools==59.5.0 +av==12.3.0 +itsdangerous==2.2.0 +wandb==0.19.1 +nest-asyncio==1.6.0 +triton==3.0.0 +nvidia-ml-py==12.560.30 +nvidia-cuda-nvrtc-cu12==12.1.105 +tomli==2.2.1 +tqdm==4.67.1 +termcolor==2.4.0 +hydra-core==1.2.0 +smmap==5.0.1 +ipython==8.12.3 +pyquaternion==0.9.9 +idna==3.10 +yacs==0.1.8 +tzdata==2024.2 +torch==2.4.1+cu121 +widgetsnbextension==4.0.13 +scikit-learn==1.3.2 +sapien==2.2.1 +nvidia-cublas-cu12==12.1.3.1 +huggingface-hub==0.21.0 +gitdb==4.0.11 +importlib_resources==6.4.5 +pydantic_core==2.27.1 +jupyter_core==5.7.2 +sentry-sdk==2.19.2 +nvidia-cudnn-cu12==9.1.0.70 +fsspec==2024.2.0 +psutil==6.1.0 +pytz==2024.2 +backcall==0.2.0 +nvidia-cusolver-cu12==11.4.5.107 +lxml==5.3.0 +blessed==1.20.0 +pure_eval==0.2.3 +diffusion_policy_3d==0.0.0 +dexart==0.1.0 +gym==0.21.0 +metaworld==0.1.0 +mj-envs==1.0.0 +mjrl==1.0.0 +pytorch3d==0.7.6 +diffusion_policy_3d==0.0.0 diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/wandb-metadata.json b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..46553bd110d943dacea4c1878305af53ac95fa10 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/files/wandb-metadata.json @@ -0,0 +1,95 @@ +{ + "os": "Linux-5.15.0-117-generic-x86_64-with-glibc2.17", + "python": "CPython 3.8.20", + "startedAt": "2024-12-18T12:53:29.016190Z", + "args": [ + "--config-name=dp3.yaml", + "task=metaworld_coffee-push", + "hydra.run.dir=data/outputs/metaworld_coffee-push-dp3-1218_seed0", + "training.debug=False", + "training.seed=0", + "training.device=cuda:0", + "exp_name=metaworld_coffee-push-dp3-1218", + "logging.mode=online", + "checkpoint.save_ckpt=True" + ], + "program": "train.py", + "codePath": "3D-Diffusion-Policy/train.py", + "git": { + "remote": "git@github.com:YanjieZe/3D-Diffusion-Policy.git", + "commit": "3153fb32c17308fc828015156480b77f6b2e666a" + }, + "email": "chenhaojun001@gmail.com", + "root": "/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0", + "host": "Server-3090", + "executable": "/home/chenhaojun/miniconda3/envs/dp3/bin/python", + "codePathLocal": "train.py", + "cpu_count": 128, + "cpu_count_logical": 256, + "gpu": "NVIDIA GeForce RTX 3090", + "gpu_count": 8, + "disk": { + "/": { + "total": "943412031488", + "used": "779944341504" + } + }, + "memory": { + "total": "1081980669952" + }, + "cpu": { + "count": 128, + "countLogical": 256 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + }, + { + "name": "NVIDIA GeForce RTX 3090", + "memoryTotal": "25769803776", + "cudaCores": 10496, + "architecture": "Ampere" + } + ], + "cudaVersion": "12.5" +} \ No newline at end of file diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-internal.log b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b6db6be1dc6d7d99f902c42a19a2be02377a8d12 --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-12-18T20:53:29.017441565+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-18T20:53:29.017455305+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-core.log"} +{"time":"2024-12-18T20:53:29.229830137+08:00","level":"INFO","msg":"created new stream","id":"ss7puy3o"} +{"time":"2024-12-18T20:53:29.229859187+08:00","level":"INFO","msg":"stream: started","id":"ss7puy3o"} +{"time":"2024-12-18T20:53:29.229930566+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:29.229981456+08:00","level":"INFO","msg":"handler: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:29.230062516+08:00","level":"INFO","msg":"sender: started","stream_id":"ss7puy3o"} +{"time":"2024-12-18T20:53:30.740244566+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-18T20:53:33.480079222+08:00","level":"WARN","msg":"handler: ignoring partial history record","step":0,"current":1} +{"time":"2024-12-19T00:16:29.479466503+08:00","level":"INFO","msg":"stream: closing","id":"ss7puy3o"} +{"time":"2024-12-19T00:16:29.479515692+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-19T00:16:29.480544547+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-19T00:16:32.191189049+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-19T00:16:34.147859526+08:00","level":"INFO","msg":"handler: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:16:34.147927026+08:00","level":"INFO","msg":"sender: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:16:34.147921096+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ss7puy3o"} +{"time":"2024-12-19T00:16:34.148083415+08:00","level":"INFO","msg":"stream: closed","id":"ss7puy3o"} diff --git a/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug.log b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f8067b04c0afc8da520b50b10dac2f1b9a62f08d --- /dev/null +++ b/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug.log @@ -0,0 +1,25 @@ +2024-12-18 20:53:28,982 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Configure stats pid to 2966139 +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug.log +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0/wandb/run-20241218_205328-ss7puy3o/logs/debug-internal.log +2024-12-18 20:53:28,983 INFO MainThread:2966139 [wandb_init.py:init():644] calling init triggers +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'coffee-push', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'coffee-push', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_coffee-push_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'coffee-push', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_coffee-push-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_coffee-push-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/20.53.26_train_dp3_coffee-push', 'wandb_name_base': '2024.12.18-20.53.26_train_dp3_coffee-push'}} +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():680] starting backend +2024-12-18 20:53:28,984 INFO MainThread:2966139 [wandb_init.py:init():684] sending inform_init request +2024-12-18 20:53:29,015 INFO MainThread:2966139 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 20:53:29,015 INFO MainThread:2966139 [wandb_init.py:init():697] backend started and connected +2024-12-18 20:53:29,022 INFO MainThread:2966139 [wandb_init.py:init():790] updated telemetry +2024-12-18 20:53:29,172 INFO MainThread:2966139 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 20:53:30,726 INFO MainThread:2966139 [wandb_init.py:init():867] run resumed +2024-12-18 20:53:30,731 INFO MainThread:2966139 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 20:53:30,858 INFO MainThread:2966139 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 20:53:30,858 INFO MainThread:2966139 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 20:53:30,859 INFO MainThread:2966139 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 20:53:30,859 INFO MainThread:2966139 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 20:53:30,860 INFO MainThread:2966139 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 20:53:30,861 INFO MainThread:2966139 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_coffee-push-dp3-1218_seed0'} +2024-12-19 00:16:29,480 WARNING MsgRouterThr:2966139 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_dial-turn-dp3-1218_seed0/train.log b/metaworld_dial-turn-dp3-1218_seed0/train.log new file mode 100644 index 0000000000000000000000000000000000000000..34cdaf06eeb4b185d0f92b83fdeaf28851caccc4 --- /dev/null +++ b/metaworld_dial-turn-dp3-1218_seed0/train.log @@ -0,0 +1 @@ +[2024-12-18 20:54:19,043][diffusion_policy_3d.model.diffusion.conditional_unet1d][INFO] - number of parameters: 2.550744e+08 diff --git a/metaworld_dial-turn-dp3-1218_seed0/wandb/debug.log b/metaworld_dial-turn-dp3-1218_seed0/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2fb994f6368bd0448de3a367d4b7620c15168310 --- /dev/null +++ b/metaworld_dial-turn-dp3-1218_seed0/wandb/debug.log @@ -0,0 +1,24 @@ +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_setup.py:_flush():68] Current SDK version is 0.19.1 +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_setup.py:_flush():68] Configure stats pid to 2971297 +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/.config/wandb/settings +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_setup.py:_flush():68] Loading settings from /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/wandb/settings +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_setup.py:_flush():68] Loading settings from environment variables +2024-12-18 20:54:27,125 INFO MainThread:2971297 [wandb_init.py:_log_setup():528] Logging user logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug.log +2024-12-18 20:54:27,126 INFO MainThread:2971297 [wandb_init.py:_log_setup():529] Logging internal logs to /home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug-internal.log +2024-12-18 20:54:27,126 INFO MainThread:2971297 [wandb_init.py:init():644] calling init triggers +2024-12-18 20:54:27,126 INFO MainThread:2971297 [wandb_init.py:init():650] wandb.init called with sweep_config: {} +config: {'task': {'name': 'dial-turn', 'task_name': 'train_dp3', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'env_runner': {'_target_': 'diffusion_policy_3d.env_runner.metaworld_runner.MetaworldRunner', 'eval_episodes': 20, 'n_obs_steps': 2, 'n_action_steps': 3, 'fps': 10, 'n_envs': None, 'n_train': None, 'n_test': None, 'task_name': 'dial-turn', 'device': 'cuda:0', 'use_point_crop': True}, 'dataset': {'_target_': 'diffusion_policy_3d.dataset.metaworld_dataset.MetaworldDataset', 'zarr_path': '/data/haojun/datasets/3d-dp/metaworld_dial-turn_expert.zarr', 'horizon': 4, 'pad_before': 1, 'pad_after': 2, 'seed': 42, 'val_ratio': 0.02, 'max_train_episodes': 90}}, 'name': 'train_dp3', 'task_name': 'dial-turn', 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'exp_name': 'metaworld_dial-turn-dp3-1218', 'horizon': 4, 'n_obs_steps': 2, 'n_action_steps': 3, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy_3d.policy.dp3.DP3', 'use_point_crop': True, 'condition_type': 'film', 'use_down_condition': True, 'use_mid_condition': True, 'use_up_condition': True, 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'crop_shape': [80, 80], 'encoder_output_dim': 64, 'horizon': 4, 'kernel_size': 5, 'n_action_steps': 3, 'n_groups': 8, 'n_obs_steps': 2, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddim.DDIMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'clip_sample': True, 'set_alpha_to_one': True, 'steps_offset': 0, 'prediction_type': 'sample'}, 'num_inference_steps': 10, 'obs_as_global_cond': True, 'shape_meta': {'obs': {'point_cloud': {'shape': [512, 3], 'type': 'point_cloud'}, 'agent_pos': {'shape': [9], 'type': 'low_dim'}}, 'action': {'shape': [4]}}, 'use_pc_color': False, 'pointnet_type': 'pointnet', 'pointcloud_encoder_cfg': {'in_channels': 3, 'out_channels': 64, 'use_layernorm': True, 'final_norm': 'layernorm', 'normal_channel': False}}, 'ema': {'_target_': 'diffusion_policy_3d.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': True, 'pin_memory': True, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 128, 'num_workers': 8, 'shuffle': False, 'pin_memory': True, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 0, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 3000, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 200, 'checkpoint_every': 200, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'group': 'metaworld_dial-turn-dp3-1218', 'id': None, 'mode': 'online', 'name': '0', 'project': 'dp3', 'resume': True, 'tags': ['dp3']}, 'checkpoint': {'save_ckpt': True, 'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.12.18/20.54.24_train_dp3_dial-turn', 'wandb_name_base': '2024.12.18-20.54.24_train_dp3_dial-turn'}} +2024-12-18 20:54:27,126 INFO MainThread:2971297 [wandb_init.py:init():680] starting backend +2024-12-18 20:54:27,126 INFO MainThread:2971297 [wandb_init.py:init():684] sending inform_init request +2024-12-18 20:54:27,155 INFO MainThread:2971297 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-18 20:54:27,156 INFO MainThread:2971297 [wandb_init.py:init():697] backend started and connected +2024-12-18 20:54:27,161 INFO MainThread:2971297 [wandb_init.py:init():790] updated telemetry +2024-12-18 20:54:27,313 INFO MainThread:2971297 [wandb_init.py:init():822] communicating run to backend with 90.0 second timeout +2024-12-18 20:54:29,018 INFO MainThread:2971297 [wandb_init.py:init():874] starting run threads in backend +2024-12-18 20:54:29,136 INFO MainThread:2971297 [wandb_run.py:_console_start():2374] atexit reg +2024-12-18 20:54:29,136 INFO MainThread:2971297 [wandb_run.py:_redirect():2224] redirect: wrap_raw +2024-12-18 20:54:29,136 INFO MainThread:2971297 [wandb_run.py:_redirect():2289] Wrapping output streams. +2024-12-18 20:54:29,136 INFO MainThread:2971297 [wandb_run.py:_redirect():2314] Redirects installed. +2024-12-18 20:54:29,139 INFO MainThread:2971297 [wandb_init.py:init():916] run started, returning control to user process +2024-12-18 20:54:29,139 INFO MainThread:2971297 [wandb_run.py:_config_callback():1279] config_cb None None {'output_dir': '/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_dial-turn-dp3-1218_seed0'} +2024-12-19 00:15:23,005 WARNING MsgRouterThr:2971297 [router.py:message_loop():75] message_loop has been closed diff --git a/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug-internal.log b/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..686541a87fd78dab2df48aa757e342d75d3df346 --- /dev/null +++ b/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug-internal.log @@ -0,0 +1,17 @@ +{"time":"2024-12-18T20:54:27.158786827+08:00","level":"INFO","msg":"using version","core version":"0.19.1"} +{"time":"2024-12-18T20:54:27.158813197+08:00","level":"INFO","msg":"created symlink","path":"/home/chenhaojun/3D-Diffusion-Policy/3D-Diffusion-Policy/data/outputs/metaworld_dial-turn-dp3-1218_seed0/wandb/run-20241218_205427-ce4joqwv/logs/debug-core.log"} +{"time":"2024-12-18T20:54:27.479425897+08:00","level":"INFO","msg":"created new stream","id":"ce4joqwv"} +{"time":"2024-12-18T20:54:27.479454887+08:00","level":"INFO","msg":"stream: started","id":"ce4joqwv"} +{"time":"2024-12-18T20:54:27.479549587+08:00","level":"INFO","msg":"writer: Do: started","stream_id":"ce4joqwv"} +{"time":"2024-12-18T20:54:27.479564937+08:00","level":"INFO","msg":"sender: started","stream_id":"ce4joqwv"} +{"time":"2024-12-18T20:54:27.481402608+08:00","level":"INFO","msg":"handler: started","stream_id":"ce4joqwv"} +{"time":"2024-12-18T20:54:29.027448033+08:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2024-12-18T22:52:46.540845094+08:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/chenhaojun001/dp3/ce4joqwv/file_stream","body":"\n\n\n502 Server Error\n\n\n

Error: Server Error

\n

The server encountered a temporary error and could not complete your request.

Please try again in 30 seconds.

\n

\n\n"} +{"time":"2024-12-19T00:15:23.004591189+08:00","level":"INFO","msg":"stream: closing","id":"ce4joqwv"} +{"time":"2024-12-19T00:15:23.004634129+08:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2024-12-19T00:15:23.005928222+08:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2024-12-19T00:15:25.912517825+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"} +{"time":"2024-12-19T00:15:27.215789037+08:00","level":"INFO","msg":"handler: closed","stream_id":"ce4joqwv"} +{"time":"2024-12-19T00:15:27.215848516+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ce4joqwv"} +{"time":"2024-12-19T00:15:27.215877416+08:00","level":"INFO","msg":"sender: closed","stream_id":"ce4joqwv"} +{"time":"2024-12-19T00:15:27.215986116+08:00","level":"INFO","msg":"stream: closed","id":"ce4joqwv"}