Delete train_1/wandb

train_1/wandb/debug-internal.log

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b77a0678bde95cb25733d66027207ceb95dd540d03bf0cf859f18363dfab5d15
-size 91074601

train_1/wandb/debug.log

@@ -1,30 +0,0 @@
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Current SDK version is 0.17.6
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Configure stats pid to 3381984
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Loading settings from /cluster/home/sigmunhh/.config/wandb/settings
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Loading settings from /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/wandb/settings
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Loading settings from environment variables: {}
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/train.py', 'program': '/tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/train.py'}
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_setup.py:_flush():76] Applying login settings: {}
-2024-11-20 22:14:47,480 INFO    MainThread:3381984 [wandb_init.py:_log_setup():521] Logging user logs to /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1/wandb/run-20241120_221447-3pyqkvto_train_1/logs/debug.log
-2024-11-20 22:14:47,481 INFO    MainThread:3381984 [wandb_init.py:_log_setup():522] Logging internal logs to /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1/wandb/run-20241120_221447-3pyqkvto_train_1/logs/debug-internal.log
-2024-11-20 22:14:47,481 INFO    MainThread:3381984 [wandb_init.py:init():559] calling init triggers
-2024-11-20 22:14:47,481 INFO    MainThread:3381984 [wandb_init.py:init():566] wandb.init called with sweep_config: {}
-config: {'name': 'train_tedi_unet_hybrid', '_target_': 'diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace', 'task_name': 'lift_image', 'shape_meta': {'obs': {'agentview_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eye_in_hand_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eef_pos': {'shape': [3]}, 'robot0_eef_quat': {'shape': [4]}, 'robot0_gripper_qpos': {'shape': [2]}}, 'action': {'shape': [10]}}, 'exp_name': 'default', 'horizon': 20, 'n_obs_steps': 2, 'n_action_steps': 8, 'n_latency_steps': 0, 'dataset_obs_steps': 2, 'past_action_visible': False, 'keypoint_visible_rate': 1.0, 'obs_as_global_cond': True, 'policy': {'_target_': 'diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy', 'shape_meta': {'obs': {'agentview_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eye_in_hand_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eef_pos': {'shape': [3]}, 'robot0_eef_quat': {'shape': [4]}, 'robot0_gripper_qpos': {'shape': [2]}}, 'action': {'shape': [10]}}, 'noise_scheduler': {'_target_': 'diffusion_policy.policy.schedulers.DDPMTEDiScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'variance_type': 'fixed_small', 'clip_sample': True, 'prediction_type': 'epsilon'}, 'horizon': 20, 'n_action_steps': 8, 'n_obs_steps': 2, 'num_inference_steps': 100, 'obs_as_global_cond': True, 'crop_shape': [76, 76], 'diffusion_step_embed_dim': 128, 'down_dims': [512, 1024, 2048], 'kernel_size': 5, 'n_groups': 8, 'cond_predict_scale': True, 'obs_encoder_group_norm': True, 'eval_fixed_crop': True, 'temporally_constant_weight': 0.2, 'temporally_increasing_weight': 0.0, 'temporally_random_weights': 0.0, 'chunk_wise_weight': 0.8, 'buffer_init': 'zero'}, 'ema': {'_target_': 'diffusion_policy.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 64, 'num_workers': 16, 'shuffle': True, 'pin_memory': True, 'persistent_workers': True}, 'val_dataloader': {'batch_size': 64, 'num_workers': 16, 'shuffle': False, 'pin_memory': True, 'persistent_workers': True}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 43, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 1501, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 50, 'checkpoint_every': 50, 'val_every': 1, 'sample_every': 5, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'project': 'diffusion_policy_debug', 'resume': True, 'mode': 'online', 'name': '2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1', 'tags': ['train_tedi_unet_hybrid', 'lift_image', 'default'], 'id': '3pyqkvto_train_1', 'group': '3pyqkvto'}, 'checkpoint': {'topk': {'monitor_key': 'test_mean_score', 'mode': 'max', 'k': 1, 'format_str': 'epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt'}, 'save_last_ckpt': False, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2024.11.20/22.14.21_train_tedi_unet_hybrid_lift_image', 'wandb_name_base': '2024.11.20-22.14.21_train_tedi_unet_hybrid_lift_image'}, 'task': {'name': 'lift_image', 'shape_meta': {'obs': {'agentview_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eye_in_hand_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eef_pos': {'shape': [3]}, 'robot0_eef_quat': {'shape': [4]}, 'robot0_gripper_qpos': {'shape': [2]}}, 'action': {'shape': [10]}}, 'task_name': 'lift', 'dataset_type': 'ph', 'dataset_path': 'data/robomimic/datasets/lift/ph/image_abs.hdf5', 'abs_action': True, 'env_runner': {'_target_': 'diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner', 'dataset_path': 'data/robomimic/datasets/lift/ph/image_abs.hdf5', 'shape_meta': {'obs': {'agentview_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eye_in_hand_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eef_pos': {'shape': [3]}, 'robot0_eef_quat': {'shape': [4]}, 'robot0_gripper_qpos': {'shape': [2]}}, 'action': {'shape': [10]}}, 'n_train': 6, 'n_train_vis': 2, 'train_start_idx': 0, 'n_test': 50, 'n_test_vis': 4, 'test_start_seed': 4400000, 'max_steps': 400, 'n_obs_steps': 2, 'n_action_steps': 8, 'render_obs_key': 'agentview_image', 'fps': 10, 'crf': 22, 'past_action': False, 'abs_action': True, 'tqdm_interval_sec': 1.0, 'n_envs': 28}, 'dataset': {'_target_': 'diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset', 'shape_meta': {'obs': {'agentview_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eye_in_hand_image': {'shape': [3, 84, 84], 'type': 'rgb'}, 'robot0_eef_pos': {'shape': [3]}, 'robot0_eef_quat': {'shape': [4]}, 'robot0_gripper_qpos': {'shape': [2]}}, 'action': {'shape': [10]}}, 'dataset_path': 'data/robomimic/datasets/lift/ph/image_abs.hdf5', 'horizon': 20, 'pad_before': 1, 'pad_after': 7, 'n_obs_steps': 2, 'abs_action': True, 'rotation_rep': 'rotation_6d', 'use_legacy_normalizer': False, 'use_cache': True, 'seed': 42, 'val_ratio': 0.02}}}
-2024-11-20 22:14:47,481 INFO    MainThread:3381984 [wandb_init.py:init():609] starting backend
-2024-11-20 22:14:47,481 INFO    MainThread:3381984 [wandb_init.py:init():613] setting up manager
-2024-11-20 22:14:47,484 INFO    MainThread:3381984 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2024-11-20 22:14:47,519 INFO    MainThread:3381984 [wandb_init.py:init():621] backend started and connected
-2024-11-20 22:14:47,524 INFO    MainThread:3381984 [wandb_init.py:init():716] updated telemetry
-2024-11-20 22:14:47,524 INFO    MainThread:3381984 [wandb_init.py:init():749] communicating run to backend with 90.0 second timeout
-2024-11-20 22:14:48,791 INFO    MainThread:3381984 [wandb_run.py:_on_init():2438] communicating current version
-2024-11-20 22:14:48,886 INFO    MainThread:3381984 [wandb_run.py:_on_init():2447] got version response upgrade_message: "wandb version 0.18.7 is available!  To upgrade, please run:\n $ pip install wandb --upgrade"
-
-2024-11-20 22:14:48,887 INFO    MainThread:3381984 [wandb_init.py:init():800] starting run threads in backend
-2024-11-20 22:14:49,182 INFO    MainThread:3381984 [wandb_run.py:_console_start():2416] atexit reg
-2024-11-20 22:14:49,182 INFO    MainThread:3381984 [wandb_run.py:_redirect():2258] redirect: wrap_raw
-2024-11-20 22:14:49,183 INFO    MainThread:3381984 [wandb_run.py:_redirect():2323] Wrapping output streams.
-2024-11-20 22:14:49,183 INFO    MainThread:3381984 [wandb_run.py:_redirect():2348] Redirects installed.
-2024-11-20 22:14:49,233 INFO    MainThread:3381984 [wandb_init.py:init():843] run started, returning control to user process
-2024-11-20 22:14:49,234 INFO    MainThread:3381984 [wandb_run.py:_config_callback():1385] config_cb None None {'output_dir': '/tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1'}
-2024-11-21 05:05:05,978 WARNING MsgRouterThr:3381984 [router.py:message_loop():77] message_loop has been closed

train_1/wandb/run-20241120_221447-3pyqkvto_train_1/files/code/train.py

@@ -1,35 +0,0 @@
-"""
-Usage:
-Training:
-python train.py --config-name=train_diffusion_lowdim_workspace
-"""
-
-import sys
-# use line-buffering for both stdout and stderr
-sys.stdout = open(sys.stdout.fileno(), mode='w', buffering=1)
-sys.stderr = open(sys.stderr.fileno(), mode='w', buffering=1)
-
-import hydra
-from omegaconf import OmegaConf
-import pathlib
-from diffusion_policy.workspace.base_workspace import BaseWorkspace
-
-# allows arbitrary python code execution in configs using the ${eval:''} resolver
-OmegaConf.register_new_resolver("eval", eval, replace=True)
-
-@hydra.main(
-    version_base=None,
-    config_path=str(pathlib.Path(__file__).parent.joinpath(
-        'diffusion_policy','config'))
-)
-def main(cfg: OmegaConf):
-    # resolve immediately so all the ${now:} resolvers
-    # will use the same time.
-    OmegaConf.resolve(cfg)
-
-    cls = hydra.utils.get_class(cfg._target_)
-    workspace: BaseWorkspace = cls(cfg)
-    workspace.run()
-
-if __name__ == "__main__":
-    main()

train_1/wandb/run-20241120_221447-3pyqkvto_train_1/files/config.yaml

@@ -1,370 +0,0 @@
-wandb_version: 1
-
-name:
-  desc: null
-  value: train_tedi_unet_hybrid
-_target_:
-  desc: null
-  value: diffusion_policy.workspace.train_tedi_unet_hybrid_workspace.TrainTEDIUnetHybridWorkspace
-task_name:
-  desc: null
-  value: lift_image
-shape_meta:
-  desc: null
-  value:
-    obs:
-      agentview_image:
-        shape:
-        - 3
-        - 84
-        - 84
-        type: rgb
-      robot0_eye_in_hand_image:
-        shape:
-        - 3
-        - 84
-        - 84
-        type: rgb
-      robot0_eef_pos:
-        shape:
-        - 3
-      robot0_eef_quat:
-        shape:
-        - 4
-      robot0_gripper_qpos:
-        shape:
-        - 2
-    action:
-      shape:
-      - 10
-exp_name:
-  desc: null
-  value: default
-horizon:
-  desc: null
-  value: 20
-n_obs_steps:
-  desc: null
-  value: 2
-n_action_steps:
-  desc: null
-  value: 8
-n_latency_steps:
-  desc: null
-  value: 0
-dataset_obs_steps:
-  desc: null
-  value: 2
-past_action_visible:
-  desc: null
-  value: false
-keypoint_visible_rate:
-  desc: null
-  value: 1.0
-obs_as_global_cond:
-  desc: null
-  value: true
-policy:
-  desc: null
-  value:
-    _target_: diffusion_policy.policy.tedi_unet_hybrid_image_policy.TEDIUnetHybridImagePolicy
-    shape_meta:
-      obs:
-        agentview_image:
-          shape:
-          - 3
-          - 84
-          - 84
-          type: rgb
-        robot0_eye_in_hand_image:
-          shape:
-          - 3
-          - 84
-          - 84
-          type: rgb
-        robot0_eef_pos:
-          shape:
-          - 3
-        robot0_eef_quat:
-          shape:
-          - 4
-        robot0_gripper_qpos:
-          shape:
-          - 2
-      action:
-        shape:
-        - 10
-    noise_scheduler:
-      _target_: diffusion_policy.policy.schedulers.DDPMTEDiScheduler
-      num_train_timesteps: 100
-      beta_start: 0.0001
-      beta_end: 0.02
-      beta_schedule: squaredcos_cap_v2
-      variance_type: fixed_small
-      clip_sample: true
-      prediction_type: epsilon
-    horizon: 20
-    n_action_steps: 8
-    n_obs_steps: 2
-    num_inference_steps: 100
-    obs_as_global_cond: true
-    crop_shape:
-    - 76
-    - 76
-    diffusion_step_embed_dim: 128
-    down_dims:
-    - 512
-    - 1024
-    - 2048
-    kernel_size: 5
-    n_groups: 8
-    cond_predict_scale: true
-    obs_encoder_group_norm: true
-    eval_fixed_crop: true
-    temporally_constant_weight: 0.2
-    temporally_increasing_weight: 0.0
-    temporally_random_weights: 0.0
-    chunk_wise_weight: 0.8
-    buffer_init: zero
-ema:
-  desc: null
-  value:
-    _target_: diffusion_policy.model.diffusion.ema_model.EMAModel
-    update_after_step: 0
-    inv_gamma: 1.0
-    power: 0.75
-    min_value: 0.0
-    max_value: 0.9999
-dataloader:
-  desc: null
-  value:
-    batch_size: 64
-    num_workers: 16
-    shuffle: true
-    pin_memory: true
-    persistent_workers: true
-val_dataloader:
-  desc: null
-  value:
-    batch_size: 64
-    num_workers: 16
-    shuffle: false
-    pin_memory: true
-    persistent_workers: true
-optimizer:
-  desc: null
-  value:
-    _target_: torch.optim.AdamW
-    lr: 0.0001
-    betas:
-    - 0.95
-    - 0.999
-    eps: 1.0e-08
-    weight_decay: 1.0e-06
-training:
-  desc: null
-  value:
-    device: cuda:0
-    seed: 43
-    debug: false
-    resume: true
-    lr_scheduler: cosine
-    lr_warmup_steps: 500
-    num_epochs: 1501
-    gradient_accumulate_every: 1
-    use_ema: true
-    rollout_every: 50
-    checkpoint_every: 50
-    val_every: 1
-    sample_every: 5
-    max_train_steps: null
-    max_val_steps: null
-    tqdm_interval_sec: 1.0
-logging:
-  desc: null
-  value:
-    project: diffusion_policy_debug
-    resume: true
-    mode: online
-    name: 2024.11.20-22.13.58_train_tedi_unet_hybrid_lift_image_train_1
-    tags:
-    - train_tedi_unet_hybrid
-    - lift_image
-    - default
-    id: 3pyqkvto_train_1
-    group: 3pyqkvto
-checkpoint:
-  desc: null
-  value:
-    topk:
-      monitor_key: test_mean_score
-      mode: max
-      k: 1
-      format_str: epoch={epoch:04d}-test_mean_score={test_mean_score:.3f}.ckpt
-    save_last_ckpt: false
-    save_last_snapshot: false
-multi_run:
-  desc: null
-  value:
-    run_dir: data/outputs/2024.11.20/22.14.21_train_tedi_unet_hybrid_lift_image
-    wandb_name_base: 2024.11.20-22.14.21_train_tedi_unet_hybrid_lift_image
-task:
-  desc: null
-  value:
-    name: lift_image
-    shape_meta:
-      obs:
-        agentview_image:
-          shape:
-          - 3
-          - 84
-          - 84
-          type: rgb
-        robot0_eye_in_hand_image:
-          shape:
-          - 3
-          - 84
-          - 84
-          type: rgb
-        robot0_eef_pos:
-          shape:
-          - 3
-        robot0_eef_quat:
-          shape:
-          - 4
-        robot0_gripper_qpos:
-          shape:
-          - 2
-      action:
-        shape:
-        - 10
-    task_name: lift
-    dataset_type: ph
-    dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
-    abs_action: true
-    env_runner:
-      _target_: diffusion_policy.env_runner.robomimic_image_runner.RobomimicImageRunner
-      dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
-      shape_meta:
-        obs:
-          agentview_image:
-            shape:
-            - 3
-            - 84
-            - 84
-            type: rgb
-          robot0_eye_in_hand_image:
-            shape:
-            - 3
-            - 84
-            - 84
-            type: rgb
-          robot0_eef_pos:
-            shape:
-            - 3
-          robot0_eef_quat:
-            shape:
-            - 4
-          robot0_gripper_qpos:
-            shape:
-            - 2
-        action:
-          shape:
-          - 10
-      n_train: 6
-      n_train_vis: 2
-      train_start_idx: 0
-      n_test: 50
-      n_test_vis: 4
-      test_start_seed: 4400000
-      max_steps: 400
-      n_obs_steps: 2
-      n_action_steps: 8
-      render_obs_key: agentview_image
-      fps: 10
-      crf: 22
-      past_action: false
-      abs_action: true
-      tqdm_interval_sec: 1.0
-      n_envs: 28
-    dataset:
-      _target_: diffusion_policy.dataset.robomimic_replay_image_dataset.RobomimicReplayImageDataset
-      shape_meta:
-        obs:
-          agentview_image:
-            shape:
-            - 3
-            - 84
-            - 84
-            type: rgb
-          robot0_eye_in_hand_image:
-            shape:
-            - 3
-            - 84
-            - 84
-            type: rgb
-          robot0_eef_pos:
-            shape:
-            - 3
-          robot0_eef_quat:
-            shape:
-            - 4
-          robot0_gripper_qpos:
-            shape:
-            - 2
-        action:
-          shape:
-          - 10
-      dataset_path: data/robomimic/datasets/lift/ph/image_abs.hdf5
-      horizon: 20
-      pad_before: 1
-      pad_after: 7
-      n_obs_steps: 2
-      abs_action: true
-      rotation_rep: rotation_6d
-      use_legacy_normalizer: false
-      use_cache: true
-      seed: 42
-      val_ratio: 0.02
-_wandb:
-  desc: null
-  value:
-    code_path: code/train.py
-    python_version: 3.9.15
-    cli_version: 0.17.6
-    framework: torch
-    is_jupyter_run: false
-    is_kaggle_kernel: false
-    start_time: 1732137287
-    t:
-      1:
-      - 1
-      - 41
-      - 49
-      - 50
-      - 55
-      - 71
-      - 83
-      2:
-      - 1
-      - 41
-      - 49
-      - 50
-      - 55
-      - 71
-      - 83
-      3:
-      - 13
-      - 14
-      - 15
-      - 16
-      - 23
-      - 61
-      4: 3.9.15
-      5: 0.17.6
-      8:
-      - 5
-      13: linux-x86_64
-output_dir:
-  desc: null
-  value: /tmp/ray/session_2024-11-20_22-13-35_079021_3380951/runtime_resources/working_dir_files/_ray_pkg_a254c17765b6da76/data/outputs/2024.11.20/22.13.58_train_tedi_unet_hybrid_lift_image/train_1