root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data replay_buffer_size: 150000 replay_buffer_num_workers: 2 nstep: 3 batch_size: 128 seed: 0 dataset_shuffle_seed: ${seed} valid_dataset_shuffle_seed: 0 train_dataset_shuffle_seed: 0 device: cuda save_video: true save_train_video: true use_tb: true use_wandb: true wandb_run_id: '63438_0' wandb_notes: 63438_0_restarted_1 eval: false eval_on_sim: true eval_on_sim_past_checkpoints: false simulation: true true_action_history: false train_pad_after: 4 process_contact_features: ${eval} obs_type: pixels use_color: true use_depth: true use_masks: false mask_list: - EE_obj_mask mask_representation: channels segmentation_model_name: gt_segmentation crop_hw: - 144 - 144 crop_down_offset: 48 color_crop_type: null depth_crop_type: null segmask_crop_type: null add_crop_binary_mask: false add_coord_conv_map: false use_context_color: false use_context_depth: false use_context_segmask: false context_color_crop_type: null context_depth_crop_type: null context_segmask_crop_type: null context_add_crop_binary_mask: false context_add_coord_conv_map: false use_contact_map: false use_sdf_maps: false use_normals_maps: false which_objects: both max_contact_prob: 1.0 max_depth: 2.0 grasped_dtc_max_value: 0.2 env_dtc_max_value: 0.4 grasped_normals_mask_max_dtc_value: 0.2 env_normals_mask_max_dtc_value: 0.4 clamp_dtc: true dtc_adaptive_normalization: false mask_normals_within_sdf: true adaptive_normals_mask: true learnable_contact_preprocess_params: false contact_model_name: gt_contact contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt encoder_type: small debug_timestamps: false open_loop: false action_trajectories: true stop_after_action: false interpolation_frequency: 25 policy_frequency: 5 wait_for_new_camera_frames: true baseline: false use_num_train_epochs_to_determine_lr_schedule: true spawn_new_env_books: false spawn_new_grasped_book: true shuffle_env_books_mode: none book_ends_dict: mode: dynamic height: 0.1 mass: 3.5 friction: 0.15 color: '#808080' record_sim_videos: true resume_wandb_run: true train_demo_idxs_list_or_num: -1 log_train_every_steps: 25 name_of_expert_demo: 240_sim_demos_left_of_4th_book_bookends_no_env_rand_20hz_act expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} store_dataset_in_memory: false expert_dataset: ${expert_dataset_dirpath}/demos.zarr action_key: action semantic_demo_grouping_name: semantic_demo_grouping.yaml semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} include_groups_list: all expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml name_of_valid_demo: 240_sim_demos_left_of_4th_book_bookends_no_env_rand_20hz_act train_num_groups: 210 valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr valid_demo_idxs_list_or_num: null val_num_groups: 30 universal_unseen_env_seed_start: 1000000 load_bc: ${agent.load_checkpoint} checkpoint_epoch_list: - 99 - 199 - 299 - 399 - 499 - 599 - 699 - 799 - 899 - 999 - 1249 - 1499 - 1749 - 1999 - 2999 - 3999 - 4999 - 5999 - 6999 - 7999 - 8999 - 9999 snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH save_snapshot: true save_last_snapshot: true save_snapshot_when_done: true top_k_checkpoints: 10 save_snapshot_link_to_weights_dir: deprecated restore_missing_checkpoint: null restore_and_eval_all_missing_checkpoints: false bc_regularize: false bc_weight_type: qfilter username: leonmkim hostname: grasp-login1 local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup local_snapshot_dir: ${local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} agent: _target_: agent.diffusion_policy.DiffusionPolicyAgent name: diffusion_policy load_checkpoint: ${eval} device: ${device} n_obs_steps: ${.config.policy_cfg.n_obs_steps} suite_name: ${suite.name} obs_type: ${obs_type} enable_arm: ${eval} enable_camera: ${eval} use_tb: ${use_tb} desired_image_shape: - 13 - 180 - 240 orig_cam_shape: - 3 - 240 - 320 actions_prenormalized: ${simulation} config: _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig compile: false device: ${device} cam_resize_shape: ${agent.desired_image_shape} orig_cam_shape: ${agent.orig_cam_shape} policy_frequency: ${policy_frequency} interpolation_frequency: ${interpolation_frequency} deterministic_actions: true deterministic_actions_seed: 0 policy_cfg: _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig n_obs_steps: 1 horizon: 28 n_action_steps: ${agent.config.policy_cfg.horizon} output_shapes: action: - 7 input_normalization_modes: observation.image: mean_std observation.state: min_max observation.action_history: min_max output_normalization_modes: action: min_max vision_backbone: resnet18 crop_distractors_tlhw: null pretrained_backbone_weights: null transforms: - _target_: torchaug.transforms.RandomAffine degrees: - -5 - 5 translate: - 0.05 - 0.05 batch_transform: true num_chunks: -1 batch_inplace: true - _target_: torchvision.transforms.v2.Grayscale num_output_channels: 3 use_group_norm: true spatial_softmax_num_keypoints: 32 action_history_encoder_config: _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig in_channels: 7 out_channels: 32 history_length: 4 kernel_size: ${agent.config.policy_cfg.kernel_size} downsample_kernel_size: 3 downsample_stride: 2 downsample_padding: 1 down_dims: - 128 - 256 - 512 kernel_size: 5 n_groups: 8 diffusion_step_embed_dim: 128 use_film_scale_modulation: true noise_scheduler_type: DDIM beta_schedule: squaredcos_cap_v2 beta_start: 0.0001 beta_end: 0.02 prediction_type: epsilon clip_sample: true clip_sample_range: 1.0 num_train_timesteps: 100 num_inference_steps: 100 do_mask_loss_for_padding: false start_horizon_at_current_step: true input_shapes: observation.image: - 13 - 180 - 240 context_observation.image: - 13 - 180 - 240 observation.state: - 8 observation.action_history: - 7 train_cfg: _target_: utils.TrainConfig lr: 0.0001 lr_scheduler: cosine lr_warmup_steps: 500 adam_betas: - 0.95 - 0.999 adam_eps: 1.0e-08 adam_weight_decay: 1.0e-06 grad_clip_norm: 10 offline_steps: 303000 use_amp: true optimizer: adamw observation_cfg: _target_: agent.encoder.VisualFeatureSet use_depth: ${use_depth} use_color: ${use_color} mask_input_dict: _target_: agent.encoder.MaskInputDict enable: ${use_masks} representation: ${mask_representation} mask_list: ${mask_list} segmentation_model_name: ${segmentation_model_name} crop_input_config: _target_: agent.encoder.CropInputConfig color_crop_type: ${color_crop_type} depth_crop_type: ${depth_crop_type} segmask_crop_type: ${segmask_crop_type} crop_hw: ${crop_hw} crop_down_offset: ${crop_down_offset} add_crop_binary_mask: ${add_crop_binary_mask} add_coord_conv_map: ${add_coord_conv_map} context_input_config: _target_: agent.encoder.ContextInputConfig use_color: ${use_context_color} use_depth: ${use_context_depth} mask_input_dict: _target_: agent.encoder.MaskInputDict enable: ${use_context_segmask} representation: ${mask_representation} mask_list: ${mask_list} segmentation_model_name: ${segmentation_model_name} crop_input_config: _target_: agent.encoder.CropInputConfig color_crop_type: ${context_color_crop_type} depth_crop_type: ${context_depth_crop_type} segmask_crop_type: ${context_segmask_crop_type} crop_hw: ${crop_hw} crop_down_offset: ${crop_down_offset} add_crop_binary_mask: ${context_add_crop_binary_mask} add_coord_conv_map: ${context_add_coord_conv_map} mask_soft_approx_scheduler_config: _target_: agent.encoder.MaskSoftApproxSchedulerConfig num_steps: 40000 initial_value: 10.0 final_value: 1000.0 interpolation_scheme: cosine use_contact_map: ${use_contact_map} use_sdf_maps: ${use_sdf_maps} use_normals_maps: ${use_normals_maps} which_objects: ${which_objects} grasped_dtc_max_value: ${grasped_dtc_max_value} env_dtc_max_value: ${env_dtc_max_value} grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} clamp_dtc: ${clamp_dtc} max_contact_prob: ${max_contact_prob} mask_normals_within_sdf: ${mask_normals_within_sdf} dtc_adaptive_normalization: ${dtc_adaptive_normalization} adaptive_normals_mask: ${adaptive_normals_mask} max_depth: ${max_depth} image_shape: ${agent.desired_image_shape} learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} learning_rate: 0.0001 weight_decay: 0.0 contact_model_name: ${contact_model_name} zero_centered: false crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} suite: suite: frankagym name: frankagym frame_stack: ${agent.n_obs_steps} action_repeat: 1 discount: 0.99 hidden_dim: 1024 num_train_frames: 1000000 num_seed_frames: 0 num_train_epochs: 1000 validate_every_epochs: 20 eval_on_sim_every_epochs: 20 validate_diffusion_on_action_loss_every_epochs: 20 eval_on_sim_every_epochs_diffusion: 500 train_eval_diffusion_on_action_loss_every_epochs: 20 check_topk_every_epochs: 20 save_snapshot_every_epochs: 25 num_seen_eval_envs: 0 num_unseen_eval_envs: 100 evals_per_env: 1 eval_every_frames: 2000 num_eval_episodes: 5 save_snapshot: true wait_for_user_to_start_episode: true task_make_fn: _target_: suite.frankagym.make name: ${task_name} height: 240 width: 320 frame_stack: ${suite.frame_stack} action_repeat: ${suite.action_repeat} seed: ${seed} enable_arm: ${agent.enable_arm} enable_gripper: ${enable_gripper} start_with_gripper_open: ${start_with_gripper_open} enable_camera: ${agent.enable_camera} path_to_depth_extrinsics: ${path_to_depth_extrinsics} contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} x_limit: ${x_limit} y_limit: ${y_limit} z_limit: ${z_limit} device: ${device} interpolation_frequency: ${interpolation_frequency} policy_frequency: ${policy_frequency} debug_timestamps: ${debug_timestamps} stop_after_action: ${stop_after_action} open_loop: ${open_loop} wait_for_new_camera_frames: ${wait_for_new_camera_frames} action_key: ${action_key} action_trajectory_horizon: ${agent.config.policy_cfg.horizon} action_trajectories: ${action_trajectories} path_to_zarr_dataset: ${expert_dataset} agent_policy_cfg: _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig compile: false device: ${device} cam_resize_shape: ${agent.desired_image_shape} orig_cam_shape: ${agent.orig_cam_shape} policy_frequency: ${policy_frequency} interpolation_frequency: ${interpolation_frequency} deterministic_actions: true deterministic_actions_seed: 0 policy_cfg: _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig n_obs_steps: 1 horizon: 28 n_action_steps: ${agent.config.policy_cfg.horizon} output_shapes: action: - 7 input_normalization_modes: observation.image: mean_std observation.state: min_max observation.action_history: min_max output_normalization_modes: action: min_max vision_backbone: resnet18 crop_distractors_tlhw: null pretrained_backbone_weights: null transforms: - _target_: torchaug.transforms.RandomAffine degrees: - -5 - 5 translate: - 0.05 - 0.05 batch_transform: true num_chunks: -1 batch_inplace: true - _target_: torchvision.transforms.v2.Grayscale num_output_channels: 3 use_group_norm: true spatial_softmax_num_keypoints: 32 action_history_encoder_config: _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig in_channels: 7 out_channels: 32 history_length: 4 kernel_size: ${agent.config.policy_cfg.kernel_size} downsample_kernel_size: 3 downsample_stride: 2 downsample_padding: 1 down_dims: - 128 - 256 - 512 kernel_size: 5 n_groups: 8 diffusion_step_embed_dim: 128 use_film_scale_modulation: true noise_scheduler_type: DDIM beta_schedule: squaredcos_cap_v2 beta_start: 0.0001 beta_end: 0.02 prediction_type: epsilon clip_sample: true clip_sample_range: 1.0 num_train_timesteps: 100 num_inference_steps: 100 do_mask_loss_for_padding: false start_horizon_at_current_step: true input_shapes: observation.image: - 13 - 180 - 240 context_observation.image: - 13 - 180 - 240 observation.state: - 8 observation.action_history: - 7 train_cfg: _target_: utils.TrainConfig lr: 0.0001 lr_scheduler: cosine lr_warmup_steps: 500 adam_betas: - 0.95 - 0.999 adam_eps: 1.0e-08 adam_weight_decay: 1.0e-06 grad_clip_norm: 10 offline_steps: 303000 use_amp: true optimizer: adamw observation_cfg: _target_: agent.encoder.VisualFeatureSet use_depth: ${use_depth} use_color: ${use_color} mask_input_dict: _target_: agent.encoder.MaskInputDict enable: ${use_masks} representation: ${mask_representation} mask_list: ${mask_list} segmentation_model_name: ${segmentation_model_name} crop_input_config: _target_: agent.encoder.CropInputConfig color_crop_type: ${color_crop_type} depth_crop_type: ${depth_crop_type} segmask_crop_type: ${segmask_crop_type} crop_hw: ${crop_hw} crop_down_offset: ${crop_down_offset} add_crop_binary_mask: ${add_crop_binary_mask} add_coord_conv_map: ${add_coord_conv_map} context_input_config: _target_: agent.encoder.ContextInputConfig use_color: ${use_context_color} use_depth: ${use_context_depth} mask_input_dict: _target_: agent.encoder.MaskInputDict enable: ${use_context_segmask} representation: ${mask_representation} mask_list: ${mask_list} segmentation_model_name: ${segmentation_model_name} crop_input_config: _target_: agent.encoder.CropInputConfig color_crop_type: ${context_color_crop_type} depth_crop_type: ${context_depth_crop_type} segmask_crop_type: ${context_segmask_crop_type} crop_hw: ${crop_hw} crop_down_offset: ${crop_down_offset} add_crop_binary_mask: ${context_add_crop_binary_mask} add_coord_conv_map: ${context_add_coord_conv_map} mask_soft_approx_scheduler_config: _target_: agent.encoder.MaskSoftApproxSchedulerConfig num_steps: 40000 initial_value: 10.0 final_value: 1000.0 interpolation_scheme: cosine use_contact_map: ${use_contact_map} use_sdf_maps: ${use_sdf_maps} use_normals_maps: ${use_normals_maps} which_objects: ${which_objects} grasped_dtc_max_value: ${grasped_dtc_max_value} env_dtc_max_value: ${env_dtc_max_value} grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} clamp_dtc: ${clamp_dtc} max_contact_prob: ${max_contact_prob} mask_normals_within_sdf: ${mask_normals_within_sdf} dtc_adaptive_normalization: ${dtc_adaptive_normalization} adaptive_normals_mask: ${adaptive_normals_mask} max_depth: ${max_depth} image_shape: ${agent.desired_image_shape} learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} learning_rate: 0.0001 weight_decay: 0.0 contact_model_name: ${contact_model_name} zero_centered: false crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} true_action_history: ${true_action_history} num_train_frames_bc: 50000 num_train_frames_drq: 1100000 stddev_schedule_drq: linear(1.0,0.1,100000) task_name: FrankaInsertion-v1 num_train_frames_vinn: 25000 num_train_frames_diffusion: 1000000 num_train_epochs_bc: 5000 num_train_epochs_diffusion: 1000 validate_every_epochs_bc: 5 validate_every_epochs_diffusion: 20 validate_diffusion_on_action_loss_every_epochs: 20 train_eval_diffusion_on_action_loss_every_epochs: 20 eval_on_sim_every_epochs_diffusion: 20 check_topk_every_epochs: 5 check_topk_every_epochs_diffusion: 20 save_snapshot_every_epochs_diffusion: 25 x_limit: - 0.2 - 0.7 y_limit: - -0.4 - 0.4 z_limit: - -0.05 - 0.55 home_displacement: - 0.55 - 0.0 - 0.55 - 180.0 - 0.0 - 0.0 enable_gripper: true start_with_gripper_open: true offset_mask: - 1 - 1 - 1 - 1 - 1 - 1 path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy test: username: leonmkim hostname: grasp-login1 name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act root_dir: ~/fish_leon expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr semantic_demo_grouping_name: semantic_demo_grouping.yaml semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} include_groups_list: all batch_size: 128 store_dataset_in_memory: false use_tb: true local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} resume_wandb_run: false feature_type: 180x240_1_RGB_D_2.0_acthist_hst4_out32_dwnkrnl3_dwnstrd2_dwnpd1