| root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data |
| replay_buffer_size: 150000 |
| replay_buffer_num_workers: 2 |
| nstep: 3 |
| batch_size: 128 |
| seed: 0 |
| dataset_shuffle_seed: ${seed} |
| valid_dataset_shuffle_seed: 0 |
| train_dataset_shuffle_seed: 0 |
| device: cuda |
| save_video: true |
| save_train_video: true |
| use_tb: true |
| use_wandb: true |
| wandb_run_id: '30579_0' |
| wandb_notes: 30579_0_req_33981_0restarted_1 |
| eval: false |
| eval_on_sim: true |
| simulation: true |
| true_action_history: false |
| train_pad_after: 4 |
| process_contact_features: ${eval} |
| obs_type: pixels |
| use_color: true |
| use_depth: true |
| use_masks: true |
| mask_list: |
| - EE_obj_mask |
| mask_representation: channels |
| segmentation_model_name: gt_segmentation |
| crop_hw: |
| - 144 |
| - 144 |
| crop_down_offset: 48 |
| color_crop_type: null |
| depth_crop_type: null |
| segmask_crop_type: null |
| add_crop_binary_mask: false |
| add_coord_conv_map: false |
| use_context_color: false |
| use_context_depth: false |
| use_context_segmask: false |
| context_color_crop_type: null |
| context_depth_crop_type: null |
| context_segmask_crop_type: null |
| context_add_crop_binary_mask: false |
| context_add_coord_conv_map: false |
| use_contact_map: true |
| use_sdf_maps: false |
| use_normals_maps: false |
| which_objects: both |
| max_contact_prob: 1.0 |
| max_depth: 2.0 |
| grasped_dtc_max_value: 0.2 |
| env_dtc_max_value: 0.4 |
| grasped_normals_mask_max_dtc_value: 0.2 |
| env_normals_mask_max_dtc_value: 0.4 |
| clamp_dtc: true |
| dtc_adaptive_normalization: false |
| mask_normals_within_sdf: true |
| adaptive_normals_mask: true |
| learnable_contact_preprocess_params: true |
| contact_model_name: gt_contact |
| contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt |
| encoder_type: small |
| debug_timestamps: false |
| open_loop: false |
| action_trajectories: true |
| stop_after_action: false |
| interpolation_frequency: 25 |
| policy_frequency: 5 |
| wait_for_new_camera_frames: true |
| baseline: false |
| train_demo_idxs_list_or_num: -1 |
| log_train_every_steps: 25 |
| name_of_expert_demo: 413_sim_demos_left_of_4th_book_20hz_act |
| expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} |
| store_dataset_in_memory: false |
| expert_dataset: ${expert_dataset_dirpath}/demos.zarr |
| action_key: action |
| semantic_demo_grouping_name: semantic_demo_grouping.yaml |
| semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} |
| include_groups_list: all |
| expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml |
| name_of_valid_demo: 413_sim_demos_left_of_4th_book_20hz_act |
| train_num_groups: 280 |
| valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr |
| valid_demo_idxs_list_or_num: null |
| val_num_groups: 30 |
| load_bc: ${agent.load_checkpoint} |
| checkpoint_epoch_list: |
| - 99 |
| - 199 |
| - 299 |
| - 399 |
| - 499 |
| - 599 |
| - 699 |
| - 799 |
| - 899 |
| - 999 |
| - 1249 |
| - 1499 |
| - 1749 |
| - 1999 |
| - 2999 |
| - 3999 |
| - 4999 |
| - 5999 |
| - 6999 |
| - 7999 |
| - 8999 |
| - 9999 |
| snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH |
| save_snapshot: true |
| save_last_snapshot: true |
| save_snapshot_when_done: true |
| top_k_checkpoints: 5 |
| save_snapshot_link_to_weights_dir: deprecated |
| restore_missing_checkpoint: null |
| bc_regularize: false |
| bc_weight_type: qfilter |
| experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} |
| agent: |
| _target_: agent.diffusion_policy.DiffusionPolicyAgent |
| name: diffusion_policy |
| load_checkpoint: ${eval} |
| device: ${device} |
| n_obs_steps: ${.config.policy_cfg.n_obs_steps} |
| suite_name: ${suite.name} |
| obs_type: ${obs_type} |
| enable_arm: ${eval} |
| enable_camera: ${eval} |
| use_tb: ${use_tb} |
| desired_image_shape: |
| - 13 |
| - 180 |
| - 240 |
| orig_cam_shape: |
| - 3 |
| - 240 |
| - 320 |
| actions_prenormalized: ${simulation} |
| config: |
| _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig |
| compile: false |
| device: ${device} |
| cam_resize_shape: ${agent.desired_image_shape} |
| orig_cam_shape: ${agent.orig_cam_shape} |
| policy_frequency: ${policy_frequency} |
| interpolation_frequency: ${interpolation_frequency} |
| policy_cfg: |
| _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig |
| n_obs_steps: 1 |
| horizon: 28 |
| n_action_steps: ${agent.config.policy_cfg.horizon} |
| output_shapes: |
| action: |
| - 7 |
| input_normalization_modes: |
| observation.image: mean_std |
| observation.state: min_max |
| observation.action_history: min_max |
| output_normalization_modes: |
| action: min_max |
| vision_backbone: resnet18 |
| crop_distractors_tlhw: null |
| pretrained_backbone_weights: null |
| transforms: |
| - _target_: torchaug.transforms.RandomAffine |
| degrees: |
| - -5 |
| - 5 |
| translate: |
| - 0.05 |
| - 0.05 |
| batch_transform: true |
| num_chunks: -1 |
| batch_inplace: true |
| - _target_: torchaug.transforms.RandomColorJitter |
| brightness: 0.3 |
| contrast: 0.4 |
| saturation: 0.5 |
| hue: 0.08 |
| batch_transform: true |
| num_chunks: -1 |
| batch_inplace: true |
| use_group_norm: true |
| spatial_softmax_num_keypoints: 32 |
| action_history_encoder_config: |
| _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig |
| in_channels: 7 |
| out_channels: 32 |
| history_length: 4 |
| kernel_size: ${agent.config.policy_cfg.kernel_size} |
| downsample_kernel_size: 3 |
| downsample_stride: 2 |
| downsample_padding: 1 |
| down_dims: |
| - 256 |
| - 512 |
| - 1024 |
| kernel_size: 5 |
| n_groups: 8 |
| diffusion_step_embed_dim: 128 |
| use_film_scale_modulation: true |
| noise_scheduler_type: DDIM |
| beta_schedule: squaredcos_cap_v2 |
| beta_start: 0.0001 |
| beta_end: 0.02 |
| prediction_type: epsilon |
| clip_sample: true |
| clip_sample_range: 1.0 |
| num_train_timesteps: 50 |
| num_inference_steps: 10 |
| do_mask_loss_for_padding: false |
| input_shapes: |
| observation.image: |
| - 13 |
| - 180 |
| - 240 |
| context_observation.image: |
| - 13 |
| - 180 |
| - 240 |
| observation.state: |
| - 8 |
| observation.action_history: |
| - 7 |
| train_cfg: |
| _target_: utils.TrainConfig |
| lr: 0.0001 |
| lr_scheduler: cosine |
| lr_warmup_steps: 500 |
| adam_betas: |
| - 0.95 |
| - 0.999 |
| adam_eps: 1.0e-08 |
| adam_weight_decay: 1.0e-06 |
| grad_clip_norm: 10 |
| offline_steps: ${num_train_frames_diffusion} |
| use_amp: true |
| observation_cfg: |
| _target_: agent.encoder.VisualFeatureSet |
| use_depth: ${use_depth} |
| use_color: ${use_color} |
| mask_input_dict: |
| _target_: agent.encoder.MaskInputDict |
| enable: ${use_masks} |
| representation: ${mask_representation} |
| mask_list: ${mask_list} |
| segmentation_model_name: ${segmentation_model_name} |
| crop_input_config: |
| _target_: agent.encoder.CropInputConfig |
| color_crop_type: ${color_crop_type} |
| depth_crop_type: ${depth_crop_type} |
| segmask_crop_type: ${segmask_crop_type} |
| crop_hw: ${crop_hw} |
| crop_down_offset: ${crop_down_offset} |
| add_crop_binary_mask: ${add_crop_binary_mask} |
| add_coord_conv_map: ${add_coord_conv_map} |
| context_input_config: |
| _target_: agent.encoder.ContextInputConfig |
| use_color: ${use_context_color} |
| use_depth: ${use_context_depth} |
| mask_input_dict: |
| _target_: agent.encoder.MaskInputDict |
| enable: ${use_context_segmask} |
| representation: ${mask_representation} |
| mask_list: ${mask_list} |
| segmentation_model_name: ${segmentation_model_name} |
| crop_input_config: |
| _target_: agent.encoder.CropInputConfig |
| color_crop_type: ${context_color_crop_type} |
| depth_crop_type: ${context_depth_crop_type} |
| segmask_crop_type: ${context_segmask_crop_type} |
| crop_hw: ${crop_hw} |
| crop_down_offset: ${crop_down_offset} |
| add_crop_binary_mask: ${context_add_crop_binary_mask} |
| add_coord_conv_map: ${context_add_coord_conv_map} |
| mask_soft_approx_scheduler_config: |
| _target_: agent.encoder.MaskSoftApproxSchedulerConfig |
| num_steps: 40000 |
| initial_value: 10.0 |
| final_value: 1000.0 |
| interpolation_scheme: cosine |
| use_contact_map: ${use_contact_map} |
| use_sdf_maps: ${use_sdf_maps} |
| use_normals_maps: ${use_normals_maps} |
| which_objects: ${which_objects} |
| grasped_dtc_max_value: ${grasped_dtc_max_value} |
| env_dtc_max_value: ${env_dtc_max_value} |
| grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} |
| env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} |
| clamp_dtc: ${clamp_dtc} |
| max_contact_prob: ${max_contact_prob} |
| mask_normals_within_sdf: ${mask_normals_within_sdf} |
| dtc_adaptive_normalization: ${dtc_adaptive_normalization} |
| adaptive_normals_mask: ${adaptive_normals_mask} |
| max_depth: ${max_depth} |
| image_shape: ${agent.desired_image_shape} |
| learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} |
| learning_rate: 0.0001 |
| weight_decay: 0.0 |
| contact_model_name: ${contact_model_name} |
| zero_centered: false |
| crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} |
| suite: |
| suite: frankagym |
| name: frankagym |
| frame_stack: ${agent.n_obs_steps} |
| action_repeat: 1 |
| discount: 0.99 |
| hidden_dim: 1024 |
| num_train_frames: 1000000 |
| num_seed_frames: 0 |
| num_train_epochs: 1000 |
| validate_every_epochs: 50 |
| eval_on_sim_every_epochs: 50 |
| validate_diffusion_on_action_loss_every_epochs: 50 |
| eval_on_sim_every_epochs_diffusion: 500 |
| train_eval_diffusion_on_action_loss_every_epochs: 50 |
| check_topk_every_epochs: 50 |
| save_snapshot_every_epochs: 100 |
| num_seen_eval_envs: 25 |
| num_unseen_eval_envs: 25 |
| evals_per_env: 1 |
| eval_every_frames: 2000 |
| num_eval_episodes: 5 |
| save_snapshot: true |
| wait_for_user_to_start_episode: true |
| task_make_fn: |
| _target_: suite.frankagym.make |
| name: ${task_name} |
| height: 240 |
| width: 320 |
| frame_stack: ${suite.frame_stack} |
| action_repeat: ${suite.action_repeat} |
| seed: ${seed} |
| enable_arm: ${agent.enable_arm} |
| enable_gripper: ${enable_gripper} |
| start_with_gripper_open: ${start_with_gripper_open} |
| enable_camera: ${agent.enable_camera} |
| path_to_depth_extrinsics: ${path_to_depth_extrinsics} |
| contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} |
| x_limit: ${x_limit} |
| y_limit: ${y_limit} |
| z_limit: ${z_limit} |
| device: ${device} |
| interpolation_frequency: ${interpolation_frequency} |
| policy_frequency: ${policy_frequency} |
| debug_timestamps: ${debug_timestamps} |
| stop_after_action: ${stop_after_action} |
| open_loop: ${open_loop} |
| wait_for_new_camera_frames: ${wait_for_new_camera_frames} |
| action_key: ${action_key} |
| action_trajectory_horizon: ${agent.config.policy_cfg.horizon} |
| action_trajectories: ${action_trajectories} |
| path_to_zarr_dataset: ${expert_dataset} |
| agent_policy_cfg: |
| _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig |
| compile: false |
| device: ${device} |
| cam_resize_shape: ${agent.desired_image_shape} |
| orig_cam_shape: ${agent.orig_cam_shape} |
| policy_frequency: ${policy_frequency} |
| interpolation_frequency: ${interpolation_frequency} |
| policy_cfg: |
| _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig |
| n_obs_steps: 1 |
| horizon: 28 |
| n_action_steps: ${agent.config.policy_cfg.horizon} |
| output_shapes: |
| action: |
| - 7 |
| input_normalization_modes: |
| observation.image: mean_std |
| observation.state: min_max |
| observation.action_history: min_max |
| output_normalization_modes: |
| action: min_max |
| vision_backbone: resnet18 |
| crop_distractors_tlhw: null |
| pretrained_backbone_weights: null |
| transforms: |
| - _target_: torchaug.transforms.RandomAffine |
| degrees: |
| - -5 |
| - 5 |
| translate: |
| - 0.05 |
| - 0.05 |
| batch_transform: true |
| num_chunks: -1 |
| batch_inplace: true |
| - _target_: torchaug.transforms.RandomColorJitter |
| brightness: 0.3 |
| contrast: 0.4 |
| saturation: 0.5 |
| hue: 0.08 |
| batch_transform: true |
| num_chunks: -1 |
| batch_inplace: true |
| use_group_norm: true |
| spatial_softmax_num_keypoints: 32 |
| action_history_encoder_config: |
| _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig |
| in_channels: 7 |
| out_channels: 32 |
| history_length: 4 |
| kernel_size: ${agent.config.policy_cfg.kernel_size} |
| downsample_kernel_size: 3 |
| downsample_stride: 2 |
| downsample_padding: 1 |
| down_dims: |
| - 256 |
| - 512 |
| - 1024 |
| kernel_size: 5 |
| n_groups: 8 |
| diffusion_step_embed_dim: 128 |
| use_film_scale_modulation: true |
| noise_scheduler_type: DDIM |
| beta_schedule: squaredcos_cap_v2 |
| beta_start: 0.0001 |
| beta_end: 0.02 |
| prediction_type: epsilon |
| clip_sample: true |
| clip_sample_range: 1.0 |
| num_train_timesteps: 50 |
| num_inference_steps: 10 |
| do_mask_loss_for_padding: false |
| input_shapes: |
| observation.image: |
| - 13 |
| - 180 |
| - 240 |
| context_observation.image: |
| - 13 |
| - 180 |
| - 240 |
| observation.state: |
| - 8 |
| observation.action_history: |
| - 7 |
| train_cfg: |
| _target_: utils.TrainConfig |
| lr: 0.0001 |
| lr_scheduler: cosine |
| lr_warmup_steps: 500 |
| adam_betas: |
| - 0.95 |
| - 0.999 |
| adam_eps: 1.0e-08 |
| adam_weight_decay: 1.0e-06 |
| grad_clip_norm: 10 |
| offline_steps: ${num_train_frames_diffusion} |
| use_amp: true |
| observation_cfg: |
| _target_: agent.encoder.VisualFeatureSet |
| use_depth: ${use_depth} |
| use_color: ${use_color} |
| mask_input_dict: |
| _target_: agent.encoder.MaskInputDict |
| enable: ${use_masks} |
| representation: ${mask_representation} |
| mask_list: ${mask_list} |
| segmentation_model_name: ${segmentation_model_name} |
| crop_input_config: |
| _target_: agent.encoder.CropInputConfig |
| color_crop_type: ${color_crop_type} |
| depth_crop_type: ${depth_crop_type} |
| segmask_crop_type: ${segmask_crop_type} |
| crop_hw: ${crop_hw} |
| crop_down_offset: ${crop_down_offset} |
| add_crop_binary_mask: ${add_crop_binary_mask} |
| add_coord_conv_map: ${add_coord_conv_map} |
| context_input_config: |
| _target_: agent.encoder.ContextInputConfig |
| use_color: ${use_context_color} |
| use_depth: ${use_context_depth} |
| mask_input_dict: |
| _target_: agent.encoder.MaskInputDict |
| enable: ${use_context_segmask} |
| representation: ${mask_representation} |
| mask_list: ${mask_list} |
| segmentation_model_name: ${segmentation_model_name} |
| crop_input_config: |
| _target_: agent.encoder.CropInputConfig |
| color_crop_type: ${context_color_crop_type} |
| depth_crop_type: ${context_depth_crop_type} |
| segmask_crop_type: ${context_segmask_crop_type} |
| crop_hw: ${crop_hw} |
| crop_down_offset: ${crop_down_offset} |
| add_crop_binary_mask: ${context_add_crop_binary_mask} |
| add_coord_conv_map: ${context_add_coord_conv_map} |
| mask_soft_approx_scheduler_config: |
| _target_: agent.encoder.MaskSoftApproxSchedulerConfig |
| num_steps: 40000 |
| initial_value: 10.0 |
| final_value: 1000.0 |
| interpolation_scheme: cosine |
| use_contact_map: ${use_contact_map} |
| use_sdf_maps: ${use_sdf_maps} |
| use_normals_maps: ${use_normals_maps} |
| which_objects: ${which_objects} |
| grasped_dtc_max_value: ${grasped_dtc_max_value} |
| env_dtc_max_value: ${env_dtc_max_value} |
| grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} |
| env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} |
| clamp_dtc: ${clamp_dtc} |
| max_contact_prob: ${max_contact_prob} |
| mask_normals_within_sdf: ${mask_normals_within_sdf} |
| dtc_adaptive_normalization: ${dtc_adaptive_normalization} |
| adaptive_normals_mask: ${adaptive_normals_mask} |
| max_depth: ${max_depth} |
| image_shape: ${agent.desired_image_shape} |
| learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} |
| learning_rate: 0.0001 |
| weight_decay: 0.0 |
| contact_model_name: ${contact_model_name} |
| zero_centered: false |
| crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} |
| true_action_history: ${true_action_history} |
| num_train_frames_bc: 50000 |
| num_train_frames_drq: 1100000 |
| stddev_schedule_drq: linear(1.0,0.1,100000) |
| task_name: FrankaInsertion-v1 |
| num_train_frames_vinn: 25000 |
| num_train_frames_diffusion: 1000000 |
| num_train_epochs_bc: 5000 |
| num_train_epochs_diffusion: 1000 |
| validate_every_epochs_bc: 5 |
| validate_every_epochs_diffusion: 50 |
| validate_diffusion_on_action_loss_every_epochs: 50 |
| train_eval_diffusion_on_action_loss_every_epochs: 50 |
| eval_on_sim_every_epochs_diffusion: 50 |
| check_topk_every_epochs: 5 |
| check_topk_every_epochs_diffusion: 50 |
| save_snapshot_every_epochs_diffusion: 100 |
| x_limit: |
| - 0.2 |
| - 0.7 |
| y_limit: |
| - -0.4 |
| - 0.4 |
| z_limit: |
| - -0.05 |
| - 0.55 |
| home_displacement: |
| - 0.55 |
| - 0.0 |
| - 0.55 |
| - 180.0 |
| - 0.0 |
| - 0.0 |
| enable_gripper: true |
| start_with_gripper_open: true |
| offset_mask: |
| - 1 |
| - 1 |
| - 1 |
| - 1 |
| - 1 |
| - 1 |
| path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy |
| test: |
| username: leonmkim |
| hostname: grasp-login1 |
| name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act |
| root_dir: ~/fish_leon |
| expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} |
| expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr |
| semantic_demo_grouping_name: semantic_demo_grouping.yaml |
| semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} |
| include_groups_list: all |
| batch_size: 128 |
| store_dataset_in_memory: false |
| use_tb: true |
| local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup |
| local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} |
| resume_wandb_run: false |
| feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_1.0_gt_contact_acthist_hst4_out32_dwnkrnl3_dwnstrd2_dwnpd1 |
| username: leonmkim |
| hostname: grasp-login1 |
| local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup |
| local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 |
|
|
