diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a7c340b819ce7061531c65ad72626967a2fda305 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 filter=lfs diff=lfs merge=lfs -text +205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 filter=lfs diff=lfs merge=lfs -text +205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 filter=lfs diff=lfs merge=lfs -text +205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 filter=lfs diff=lfs merge=lfs -text +205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 filter=lfs diff=lfs merge=lfs -text +205317/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205317/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205317/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205317/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205317/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +205317/episode_rosbags/episode_4_2024-12-29-20-57-31.bag filter=lfs diff=lfs merge=lfs -text +205317/episode_rosbags/episode_1_2024-12-29-20-55-29.bag filter=lfs diff=lfs merge=lfs -text +205317/episode_rosbags/episode_2_2024-12-29-20-56-10.bag filter=lfs diff=lfs merge=lfs -text +205317/episode_rosbags/episode_3_2024-12-29-20-56-49.bag filter=lfs diff=lfs merge=lfs -text +205317/episode_rosbags/episode_0_2024-12-29-20-54-40.bag filter=lfs diff=lfs merge=lfs -text diff --git a/205131/.hydra/config.yaml b/205131/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8134e87aa29bc0cffb831c710cee9947c3a55525 --- /dev/null +++ b/205131/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1045_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205131/.hydra/hydra.yaml b/205131/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d570c86f417e36c94dbb2ee145e6b5760e5fa30 --- /dev/null +++ b/205131/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205131 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205131/.hydra/overrides.yaml b/205131/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205131/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205131/eval_policy.log b/205131/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..cf0b631d39396f89a7a47b4fb70db7a056db6aee --- /dev/null +++ b/205131/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-29 20:51:31,071][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 20:51:31,075][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-29 20:51:34,320][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/205132/.hydra/config.yaml b/205132/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8134e87aa29bc0cffb831c710cee9947c3a55525 --- /dev/null +++ b/205132/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1045_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205132/.hydra/hydra.yaml b/205132/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f89f8219b5743845efa30414b54fcc2a6412788 --- /dev/null +++ b/205132/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205132 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205132/.hydra/overrides.yaml b/205132/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205132/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205132/eval_robot.log b/205132/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ddfa20d9a79bd84390c2f2b55be7598009f136ce --- /dev/null +++ b/205132/eval_robot.log @@ -0,0 +1,13 @@ +[2024-12-29 20:51:32,787][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 20:51:32,790][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-29 20:51:33,311][rospy.core][INFO] - signal_shutdown [atexit] diff --git a/205317/.hydra/config.yaml b/205317/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8134e87aa29bc0cffb831c710cee9947c3a55525 --- /dev/null +++ b/205317/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1045_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${now:%H%M%S} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/205317/.hydra/hydra.yaml b/205317/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf5f9fc08fcf53f0cec9554c36212b5fb7aeb642 --- /dev/null +++ b/205317/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/205317/.hydra/overrides.yaml b/205317/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/205317/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/205317/episode_rosbags/aligned_depth_to_color_K.npy b/205317/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205317/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205317/episode_rosbags/cam_tf_world.npy b/205317/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/205317/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/205317/episode_rosbags/color_K.npy b/205317/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/205317/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/205317/episode_rosbags/depth_K.npy b/205317/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/205317/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/205317/episode_rosbags/episode_0_2024-12-29-20-54-40.bag b/205317/episode_rosbags/episode_0_2024-12-29-20-54-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..29df8444f2a539739221ccf13ec7f1d3acb05a42 --- /dev/null +++ b/205317/episode_rosbags/episode_0_2024-12-29-20-54-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263d6ba64875ad3e5c9380992b1ae9ac5466595aba489e10a184139ca4012da9 +size 1329494909 diff --git a/205317/episode_rosbags/episode_1_2024-12-29-20-55-29.bag b/205317/episode_rosbags/episode_1_2024-12-29-20-55-29.bag new file mode 100644 index 0000000000000000000000000000000000000000..6f808c91f0a47d54adfcea8322e2f70b2c30412a --- /dev/null +++ b/205317/episode_rosbags/episode_1_2024-12-29-20-55-29.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a5a4219be5162013cee691ff311f9a727cad82b88d356453a7fc8c13f6ad650 +size 1329599493 diff --git a/205317/episode_rosbags/episode_2_2024-12-29-20-56-10.bag b/205317/episode_rosbags/episode_2_2024-12-29-20-56-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..2593da6c03357e107a6dc9c4be591b460bef46a1 --- /dev/null +++ b/205317/episode_rosbags/episode_2_2024-12-29-20-56-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78194c48348d8759d08fca04bd5f3115792307c2b9618c258e792bad42c075f3 +size 1333012907 diff --git a/205317/episode_rosbags/episode_3_2024-12-29-20-56-49.bag b/205317/episode_rosbags/episode_3_2024-12-29-20-56-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..4fe6f1e51c21a3a2a4692a93db2c6829d91bf51a --- /dev/null +++ b/205317/episode_rosbags/episode_3_2024-12-29-20-56-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:078503c64c23429f55328a6c2c550475f7561dca7fcf620a4e33a7346816978e +size 1327707902 diff --git a/205317/episode_rosbags/episode_4_2024-12-29-20-57-31.bag b/205317/episode_rosbags/episode_4_2024-12-29-20-57-31.bag new file mode 100644 index 0000000000000000000000000000000000000000..fe7bf50d8b5878c16768b1ff402d52511f292447 --- /dev/null +++ b/205317/episode_rosbags/episode_4_2024-12-29-20-57-31.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:975a676b531d6e8139daea375e7609373d248fda8540766565f7604c286a86e3 +size 1333042300 diff --git a/205317/eval_robot.log b/205317/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..cbf9a5f736c4dec874bc8925bf714cc5477d9834 --- /dev/null +++ b/205317/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-29 20:53:17,250][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-29 20:53:17,254][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/205317/eval_video/0_eval.mp4 b/205317/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e30b8f25539ba150b9a1eb06bb55373125f336f0 --- /dev/null +++ b/205317/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd247d1c4c04fa7cea7a03c76f9bcf0fac708360da8e1551f468b1b47fc66b9 +size 1171585 diff --git a/205317/eval_video/1_eval.mp4 b/205317/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4425d998fb01594171bfd91b1cecddc8ba32d0aa --- /dev/null +++ b/205317/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35843717fd499551c4b0926cca736afc0945a08b7388201dffb14db7ec3fefa +size 1166652 diff --git a/205317/eval_video/2_eval.mp4 b/205317/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa2d767c8503e3550834aae742bd058f03234bb7 --- /dev/null +++ b/205317/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03a74d7b4bcab766fc29ee131b9e9ac693b435c897bde76d32747b19fb58257 +size 1184865 diff --git a/205317/eval_video/3_eval.mp4 b/205317/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ba4f985c7115dec0a37f20b9d2516228ab7524e --- /dev/null +++ b/205317/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94417ff346f07f5fa23e9c53f6b0137c96c29dcb3d88023f1ed9c85959c133e3 +size 1210929 diff --git a/205317/eval_video/4_eval.mp4 b/205317/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49141e9c64083e0f46d4c622006743c16480f4ed --- /dev/null +++ b/205317/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ea9dbf78cd7cd19362e85b72b43f16e349294c086d44f5e3adc1fd73292d66 +size 1225749 diff --git a/205317/tb/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 b/205317/tb/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 new file mode 100644 index 0000000000000000000000000000000000000000..d3a0b4ad144d0bc061ca9008297852aca182117e --- /dev/null +++ b/205317/tb/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35fcfee4a918882d02da32f4394ab050801d8e7f028ae8272dd24cece614507 +size 1103 diff --git a/205317/wandb/debug-internal.log b/205317/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..009547cb9cd45da872590b93cf6fcb48edcd6350 --- /dev/null +++ b/205317/wandb/debug-internal.log @@ -0,0 +1,749 @@ +2024-12-29 20:53:23,749 INFO StreamThr :1355438 [internal.py:wandb_internal():85] W&B internal server running at pid: 1355438, started at: 2024-12-29 20:53:23.749408 +2024-12-29 20:53:23,750 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status +2024-12-29 20:53:23,762 INFO WriterThread:1355438 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb +2024-12-29 20:53:23,762 DEBUG SenderThread:1355438 [sender.py:send():379] send: header +2024-12-29 20:53:23,770 DEBUG SenderThread:1355438 [sender.py:send():379] send: run +2024-12-29 20:53:23,990 INFO SenderThread:1355438 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files +2024-12-29 20:53:23,990 INFO SenderThread:1355438 [sender.py:_start_run_threads():1188] run started: 2n31umej with start time 1735523603.750328 +2024-12-29 20:53:24,005 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 20:53:24,005 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: check_version +2024-12-29 20:53:24,122 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 20:53:24,127 DEBUG HandlerThread:1355438 [system_info.py:__init__():26] System info init +2024-12-29 20:53:24,127 DEBUG HandlerThread:1355438 [system_info.py:__init__():41] System info init done +2024-12-29 20:53:24,127 INFO HandlerThread:1355438 [system_monitor.py:start():194] Starting system monitor +2024-12-29 20:53:24,127 INFO SystemMonitor:1355438 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 20:53:24,128 INFO HandlerThread:1355438 [system_monitor.py:probe():214] Collecting system info +2024-12-29 20:53:24,128 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 20:53:24,129 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started disk monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started memory monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started network monitoring +2024-12-29 20:53:24,140 DEBUG HandlerThread:1355438 [system_info.py:probe():152] Probing system +2024-12-29 20:53:24,143 DEBUG HandlerThread:1355438 [system_info.py:_probe_git():137] Probing git +2024-12-29 20:53:24,149 DEBUG HandlerThread:1355438 [system_info.py:_probe_git():145] Probing git done +2024-12-29 20:53:24,149 DEBUG HandlerThread:1355438 [system_info.py:probe():200] Probing system done +2024-12-29 20:53:24,150 DEBUG HandlerThread:1355438 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T01:53:24.140645', 'startedAt': '2024-12-30T01:53:23.741358', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3933.782375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.128, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.933, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 463.0247688293457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 20:53:24,150 INFO HandlerThread:1355438 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 20:53:24,150 INFO HandlerThread:1355438 [system_monitor.py:probe():227] Publishing system info +2024-12-29 20:53:24,150 DEBUG HandlerThread:1355438 [system_info.py:_save_code():44] Saving code +2024-12-29 20:53:24,151 DEBUG HandlerThread:1355438 [system_info.py:_save_code():67] Saving code done +2024-12-29 20:53:24,151 DEBUG HandlerThread:1355438 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 20:53:24,221 DEBUG HandlerThread:1355438 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 20:53:24,222 INFO HandlerThread:1355438 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 20:53:24,225 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 20:53:24,459 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 20:53:24,459 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: python_packages +2024-12-29 20:53:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:24,462 INFO HandlerThread:1355438 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/tb" + save: true +} +control { + relay_id: "4ec06bb39130473e991b7d721c6b160e" +} +_info { + stream_id: "2n31umej" +} + +2024-12-29 20:53:24,462 DEBUG HandlerThread:1355438 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 20:53:24,473 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/hlo71ln1-wandb-metadata.json +2024-12-29 20:53:24,476 INFO wandb-upload_1:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/gq25rdy7-code/FISH/eval_robot.py +2024-12-29 20:53:24,493 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/cdppaczc-diff.patch +2024-12-29 20:53:24,565 DEBUG SenderThread:1355438 [sender.py:send():379] send: telemetry +2024-12-29 20:53:24,565 DEBUG SenderThread:1355438 [sender.py:send():379] send: tbrecord +2024-12-29 20:53:24,566 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:53:24,566 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 with policy live +2024-12-29 20:53:24,873 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/9djvv8q2-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:53:24,991 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code +2024-12-29 20:53:25,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:26,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:26,991 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:27,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:28,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:29,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:29,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:30,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:30,488 DEBUG SenderThread:1355438 [sender.py:send():379] send: config +2024-12-29 20:53:31,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:32,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:32,993 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:33,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:34,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:34,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:34,993 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:35,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:36,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:36,994 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:37,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:38,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:38,995 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:39,459 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:39,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:39,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:39,622 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:40,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:40,995 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:41,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:42,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:43,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:44,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:45,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:45,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:46,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:47,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:48,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:49,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:50,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:51,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:51,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:52,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:53,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:54,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:54,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:55,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:56,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:56,527 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:56,999 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml +2024-12-29 20:53:57,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:58,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:59,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:00,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:01,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:02,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:02,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:03,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:04,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:05,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:06,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:07,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:07,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:08,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:09,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:09,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:09,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:10,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:11,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:12,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:12,558 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:13,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:14,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:15,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:16,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:17,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:18,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:18,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:19,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:20,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:21,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:22,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:23,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:23,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:24,131 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 20:54:24,131 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:54:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:24,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:25,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:26,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:27,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:28,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:28,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:29,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:30,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:31,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:32,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:33,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:34,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:34,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:35,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:36,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:37,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:38,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:39,011 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:39,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:39,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:39,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:39,548 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:40,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:41,012 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:41,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:42,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:43,013 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:43,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:44,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:45,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:45,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:46,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:47,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:48,473 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:49,473 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:50,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:50,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:51,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:52,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:53,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:54,132 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:54:54,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:54,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:55,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:55,572 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:56,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:57,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:58,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:59,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:00,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:01,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:01,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:02,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:03,018 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:03,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:04,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:05,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:06,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:06,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:07,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:08,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:08,697 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:55:08,698 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 with policy now +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:55:08,699 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:55:08,976 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/j79b7238-media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media +2024-12-29 20:55:09,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos +2024-12-29 20:55:09,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:55:09,330 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/qjvpmhlf-media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:09,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:09,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:10,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:11,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:11,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:11,599 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:12,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:13,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:14,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:15,022 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:15,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:16,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:17,023 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:17,061 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:17,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:18,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:19,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:20,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:21,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:22,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:22,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:23,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:24,132 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:55:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:24,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:25,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:26,026 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:55:26,435 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/7e1on89c-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:55:26,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:27,026 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:27,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:28,285 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:28,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:29,027 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:29,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:30,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:31,028 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:31,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:32,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:33,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:33,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:34,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:35,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:36,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:37,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:38,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:38,585 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:39,030 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:39,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:39,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:39,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:40,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:41,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:42,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:43,032 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:43,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:43,989 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:44,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:45,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:46,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:47,033 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:47,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:48,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:49,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:49,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:50,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:51,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:52,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:53,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:54,133 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:55:54,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:54,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:54,596 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:55,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:56,496 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:56,793 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:55:56,794 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 with policy now +2024-12-29 20:55:56,794 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:55:56,795 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:55:56,795 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:55:56,795 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:55:57,085 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/vqbofts6-media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,378 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/px30hwfz-media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,496 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:58,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:59,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:59,497 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:00,326 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:00,498 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:01,500 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:02,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:03,039 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:03,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:04,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:05,039 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:05,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:05,503 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:06,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:06,952 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:06,952 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:06,952 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:07,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:08,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:09,041 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:09,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:09,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:09,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:10,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:10,743 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:11,041 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:11,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:12,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:13,042 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:13,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:14,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:15,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:16,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:16,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:17,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:18,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:19,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:20,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:21,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:21,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:22,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:23,045 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:23,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:24,133 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:56:24,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:24,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:24,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:25,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:26,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:26,602 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:27,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:28,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:29,047 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:29,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:30,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:31,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:32,185 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:32,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:33,049 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:33,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:34,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:35,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:36,509 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:37,511 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:37,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:37,891 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:56:37,891 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 with policy now +2024-12-29 20:56:37,897 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:56:37,898 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:37,898 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:37,898 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:56:38,125 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/80bprm7k-media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,413 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/aun1d2pr-media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,520 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:39,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:39,480 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:39,480 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:39,521 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:40,517 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:41,520 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:42,534 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:43,052 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:43,197 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:43,535 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:44,532 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:45,053 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:45,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:46,053 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:46,307 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/tyopdo6f-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:46,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:46,784 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:46,784 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:46,784 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:47,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:47,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:47,540 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:48,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:49,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:49,093 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:49,541 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:50,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:51,055 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:51,541 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:52,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:53,056 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:53,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:54,134 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:56:54,134 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:54,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:54,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:54,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:55,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:56,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:57,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:58,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:59,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:59,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:00,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:01,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:02,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:03,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:04,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:04,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:05,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:06,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:07,577 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:08,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:09,061 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:09,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:09,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:09,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:09,611 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:10,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:11,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:12,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:13,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:14,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:15,063 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:15,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:15,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:16,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:17,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:18,549 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:19,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:19,344 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:57:19,344 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 with policy now +2024-12-29 20:57:19,347 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:57:19,347 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:57:19,347 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:57:19,347 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:57:19,550 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:19,568 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/qnlxi08l-media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:57:20,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:57:20,366 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/l9ny6g1n-media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:20,551 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:21,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:21,552 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:22,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:22,558 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:23,556 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:24,134 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:57:24,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:24,476 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:24,556 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:25,067 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:25,551 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:25,634 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:26,554 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:27,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:27,554 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:27,831 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:57:27,831 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:57:27,832 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:57:28,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:57:28,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:57:28,562 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:29,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:29,560 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:30,557 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:31,069 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:31,477 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:31,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:32,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:33,070 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:33,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:34,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:35,070 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:35,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:36,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:37,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:37,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:38,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:39,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:39,467 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:39,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:40,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:41,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:42,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:42,616 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:43,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:44,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:45,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:46,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:47,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:48,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:48,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:49,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:50,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:51,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:52,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:53,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:54,135 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:57:54,136 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:54,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:54,467 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:54,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:55,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:56,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:57,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:58,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:59,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:59,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:00,569 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:01,079 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:01,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:01,655 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:58:01,655 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 with policy now +2024-12-29 20:58:01,658 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:58:01,658 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:01,659 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:01,659 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:01,912 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/hzibu7kv-media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:58:02,358 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/m967u2jl-media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,605 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:03,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:03,606 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:04,081 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:04,483 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:04,600 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:05,610 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:06,632 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:07,081 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:07,620 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:08,625 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:09,082 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:09,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:58:09,475 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:58:09,573 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:09,614 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:10,337 DEBUG SenderThread:1355438 [sender.py:send():379] send: exit +2024-12-29 20:58:10,337 INFO SenderThread:1355438 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 20:58:10,337 INFO SenderThread:1355438 [sender.py:send_exit():588] handling runtime: 286 +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:send_exit():594] send defer +2024-12-29 20:58:10,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,338 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 20:58:10,338 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 1 +2024-12-29 20:58:10,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,338 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 20:58:10,339 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,339 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 20:58:10,339 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 2 +2024-12-29 20:58:10,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 20:58:10,339 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 20:58:10,339 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 20:58:10,388 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 20:58:10,389 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 20:58:10,389 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined network monitor +2024-12-29 20:58:10,389 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,389 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 20:58:10,389 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 3 +2024-12-29 20:58:10,389 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:58:10,389 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,390 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 20:58:10,390 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,390 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 20:58:10,390 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 4 +2024-12-29 20:58:10,390 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,390 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:11,336 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/q3m3qog2-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:16,392 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 20:58:16,392 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 20:58:16,393 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 5 +2024-12-29 20:58:16,393 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:16,393 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,393 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:16,393 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 20:58:16,393 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:16,397 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send():379] send: summary +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 6 +2024-12-29 20:58:16,398 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,398 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 20:58:16,399 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 7 +2024-12-29 20:58:16,399 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:16,399 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,399 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 20:58:16,399 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,399 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 20:58:17,085 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:17,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:17,357 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 8 +2024-12-29 20:58:17,357 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:17,357 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:17,357 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 20:58:17,358 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:17,358 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 20:58:17,358 INFO SenderThread:1355438 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 20:58:17,360 INFO SenderThread:1355438 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 20:58:17,376 INFO SenderThread:1355438 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 20:58:17,387 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 9 +2024-12-29 20:58:17,387 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:17,387 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 20:58:17,387 DEBUG SenderThread:1355438 [sender.py:send():379] send: artifact +2024-12-29 20:58:18,021 INFO wandb-upload_1:1355438 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0szy62ux +2024-12-29 20:58:18,085 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:18,124 INFO wandb-upload_0:1355438 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgsnxelt6 +2024-12-29 20:58:18,255 INFO wandb-upload_2:1355438 [upload_job.py:push():88] Uploaded file /tmp/tmpf0ewpl8h/wandb-job.json +2024-12-29 20:58:18,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA1NjUzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzQyNDMyMw==', 'versionIndex': 385}}} +2024-12-29 20:58:18,817 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml config.yaml +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json wandb-summary.json +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:19,088 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt requirements.txt +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json wandb-metadata.json +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log output.log +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch diff.patch +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 10 +2024-12-29 20:58:19,093 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,093 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,093 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 20:58:19,093 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 20:58:19,264 INFO wandb-upload_1:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml +2024-12-29 20:58:19,266 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:19,300 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt +2024-12-29 20:58:19,330 INFO wandb-upload_3:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:19,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,340 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,531 INFO Thread-11 (_thread_body):1355438 [sender.py:transition_state():614] send defer: 11 +2024-12-29 20:58:19,531 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,531 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 20:58:19,531 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,531 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 20:58:19,531 INFO SenderThread:1355438 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 20:58:19,532 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 12 +2024-12-29 20:58:19,532 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,532 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 20:58:19,532 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,532 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 20:58:19,533 INFO SenderThread:1355438 [file_stream.py:finish():601] file stream finish called +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [file_stream.py:finish():605] file stream finish is done +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 13 +2024-12-29 20:58:19,591 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,591 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 20:58:19,591 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 14 +2024-12-29 20:58:19,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send():379] send: final +2024-12-29 20:58:19,592 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send():379] send: footer +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,592 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 20:58:19,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,593 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 20:58:19,593 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: server_info +2024-12-29 20:58:19,593 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,594 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 20:58:19,594 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 20:58:19,595 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:19,682 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 20:58:19,683 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 20:58:19,683 INFO HandlerThread:1355438 [handler.py:finish():882] shutting down handler +2024-12-29 20:58:20,594 INFO WriterThread:1355438 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [sender.py:finish():1615] shutting down sender +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [file_pusher.py:join():175] waiting for file pusher diff --git a/205317/wandb/debug.log b/205317/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1e022ee97b2e3d60795a66ad04615295dfde2c73 --- /dev/null +++ b/205317/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Configure stats pid to 1355277 +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/settings +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/logs/debug.log +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/logs/debug-internal.log +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():569] calling init triggers +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 5, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1045_0', 'wandb_notes': '1045_0_', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['greece_twodim_nominal', 'greece_twodim_recovery'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317'} +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():619] starting backend +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():623] setting up manager +2024-12-29 20:53:23,748 INFO MainThread:1355277 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 20:53:23,750 INFO MainThread:1355277 [wandb_init.py:init():631] backend started and connected +2024-12-29 20:53:23,761 INFO MainThread:1355277 [wandb_init.py:init():720] updated telemetry +2024-12-29 20:53:23,767 INFO MainThread:1355277 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 20:53:24,005 INFO MainThread:1355277 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 20:53:24,117 INFO MainThread:1355277 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 20:53:24,117 INFO MainThread:1355277 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 20:53:24,461 INFO MainThread:1355277 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 20:53:24,462 INFO MainThread:1355277 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/tb, True +2024-12-29 20:53:30,487 INFO MainThread:1355277 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 20:58:20,684 WARNING MsgRouterThr:1355277 [router.py:message_loop():77] message_loop has been closed diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py b/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml b/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..690b9bde38ff207a53a38f6a0739aba713492ec3 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 5 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1045_0' +wandb_notes: + desc: null + value: '1045_0_' +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: false +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - greece_twodim_nominal + - greece_twodim_recovery +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 0 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1735523603 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch b/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..16508b376388e2a00201c6374458433cdace7f3c --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch @@ -0,0 +1,192 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 2303343..d3678fb 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -71,7 +71,7 @@ dtc_adaptive_normalization: false + mask_normals_within_sdf: true + adaptive_normals_mask: true + learnable_contact_preprocess_params: false +-contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' ++contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9' + # contact_model_name: 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9' + contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt' # no mask no flow, just context + # contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/197406_2/checkpoints/epoch=08-val_loss=0.00.ckpt' # w/ mask no flow, just context +@@ -123,6 +123,10 @@ bc_weight_type: 'qfilter' # linear, qfilter + + # Load weights + load_checkpoint: ${agent.load_checkpoint} ++# 120 demos ++wandb_run_id: '1045_0' # greece ++ ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +@@ -144,7 +148,7 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '523_0' # fowlers + # wandb_run_id: '524_0' # lib + # wandb_run_id: '525_0' # modelsys +-wandb_run_id: '526_0' # electrodyn ++# wandb_run_id: '526_0' # electrodyn + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 32 held out demos +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index 0f225ec..4d768b8 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -15,8 +15,8 @@ if not local_checkpoint_root_dir.exists(): + local_checkpoint_root_dir.mkdir(parents=True) + + # Download the model checkpoints +-# remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') +-remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++remote_checkpoint_root_dir = Path('/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') ++# remote_checkpoint_root_dir = Path('/mnt/kostas-graid/datasets/extrinsic_contact_data/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1') + remote_checkpoint_root_dir = remote_checkpoint_root_dir.expanduser() + remote_username = 'leonmkim' + remote_host = 'grasp-login1' +@@ -146,6 +146,11 @@ remote_host = 'grasp-login1' + # '265627', + # '265620', + # ] ++ ++run_id_list = [ ++ '1045_0', ++] ++ + checkpoint_epoch = 12000 + + # use subprocess to download the model checkpoints in parallel +Submodule ros_ws/src/contact_estimation_ros contains modified content +diff --git a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +index dcbcfa4..35dd7e8 100755 +--- a/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py ++++ b/ros_ws/src/contact_estimation_ros/fish_data_collection/src/rosbag_handler.py +@@ -15,6 +15,8 @@ import time + import datetime + import shutil + ++from scipy.spatial.transform import Rotation as R ++ + # solution to launching from main thread from: https://answers.ros.org/question/260212/start-launchfile-from-service/ + # from Queue import Queue + +@@ -23,7 +25,7 @@ from fish_data_collection.srv import StartBagging, StartBaggingRequest, StartBag + from sensor_msgs.msg import CameraInfo + + class RosbaggingHandler(): +- def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, camera_calibration_dir, ++ def __init__(self, base_dset_dir, experiment_dirname, bag_launch_path, demo_dirname, + multicam=False, align_depth_to_color=False): + ## bag files and target files should be identified by episode id + self.align_depth_to_color = align_depth_to_color +@@ -33,8 +35,6 @@ class RosbaggingHandler(): + self.intrinsics_saved = False + self.extrinsics_saved = False + +- self.camera_calibration_dir = camera_calibration_dir +- + self.full_bag_dir = os.path.join(self.base_dset_dir, self.experiment_dirname, self.demo_dirname) + os.makedirs(self.full_bag_dir, exist_ok=True) + +@@ -63,7 +63,7 @@ class RosbaggingHandler(): + self.abort_bagging_request = False + + self.save_camera_intrinsics() +- self.save_camera_extrinsics(self.camera_calibration_dir) ++ self.save_camera_extrinsics() + + while not rospy.is_shutdown(): + if self.start_bagging_request: +@@ -83,23 +83,56 @@ class RosbaggingHandler(): + self.start_bagging_request = True + return response + +- def save_camera_extrinsics(self, path_to_calibration_dir): ++ def save_camera_extrinsics(self): + if not self.extrinsics_saved: +- # copy over the contents of the calibration directory, but not the directory itself, into the bag directory +- # list the contents of the calibration directory that end in .npy +- calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) +- # calibration_dir_contents = os.listdir(path_to_calibration_dir) +- # copy the contents of the calibration directory into the bag directory +- for item in calibration_dir_contents: +- # make sure the item is a file +- if os.path.isfile(item): +- # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) +- shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) +- # also save the path to the calibration directory as a txt file +- with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: +- f.write(path_to_calibration_dir) ++ # get cam extrinsic ++ # use ros tf2 to get transform from color_optical_frame to panda_link0 ++ import tf2_ros ++ tfBuffer = tf2_ros.Buffer() ++ listener = tf2_ros.TransformListener(tfBuffer) ++ ++ # get transform from color_optical_frame to panda_link0 ++ # target, source where target is the parent frame and source is the child frame ++ color_tf_world_ros = tfBuffer.lookup_transform('camera_color_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.color_tf_world = np.eye(4) ++ self.color_tf_world[0, 3] =color_tf_world_ros.translation.x ++ self.color_tf_world[1, 3] =color_tf_world_ros.translation.y ++ self.color_tf_world[2, 3] =color_tf_world_ros.translation.z ++ self.color_tf_world[:3, :3] = R.from_quat([color_tf_world_ros.rotation.x, color_tf_world_ros.rotation.y, color_tf_world_ros.rotation.z, color_tf_world_ros.rotation.w]).as_matrix() ++ ++ # dump the info to the bag directory ++ np.save(os.path.join(self.full_bag_dir, 'color_tf_world.npy'), self.color_tf_world) ++ ++ depth_tf_world_ros = tfBuffer.lookup_transform('camera_depth_optical_frame', 'panda_link0', rospy.Time(0), rospy.Duration(1.0)).transform ++ self.depth_tf_world = np.eye(4) ++ self.depth_tf_world[0, 3] = depth_tf_world_ros.translation.x ++ self.depth_tf_world[1, 3] = depth_tf_world_ros.translation.y ++ self.depth_tf_world[2, 3] = depth_tf_world_ros.translation.z ++ self.depth_tf_world[:3, :3] = \ ++ R.from_quat([ ++ depth_tf_world_ros.rotation.x, ++ depth_tf_world_ros.rotation.y, ++ depth_tf_world_ros.rotation.z, ++ depth_tf_world_ros.rotation.w, ++ ]).as_matrix() ++ ++ np.save(os.path.join(self.full_bag_dir, 'depth_tf_world.npy'), self.depth_tf_world) ++ ++ # # copy over the contents of the calibration directory, but not the directory itself, into the bag directory ++ # # list the contents of the calibration directory that end in .npy ++ # calibration_dir_contents = glob.glob(os.path.join(path_to_calibration_dir, '*.npy')) ++ # # calibration_dir_contents = os.listdir(path_to_calibration_dir) ++ # # copy the contents of the calibration directory into the bag directory ++ # for item in calibration_dir_contents: ++ # # make sure the item is a file ++ # if os.path.isfile(item): ++ # # print('copying {} to {}'.format(item, os.path.join(self.full_bag_dir, os.path.basename(item)))) ++ # shutil.copy(item, os.path.join(self.full_bag_dir, os.path.basename(item))) ++ # # also save the path to the calibration directory as a txt file ++ # with open(os.path.join(self.full_bag_dir, 'calibration_dir.txt'), 'w') as f: ++ # f.write(path_to_calibration_dir) + +- self.extrinsics_saved = True ++ # self.extrinsics_saved = True + + def save_camera_intrinsics(self): + if not self.intrinsics_saved: +@@ -198,9 +231,6 @@ class RosbaggingHandler(): + if self.bag_recording: + self.current_recording_bag_path = bag_path + # response.success = True +- +- +- + # return response + + def stop_bagging_service_callback(self, req): +@@ -304,7 +334,7 @@ if __name__ == '__main__': + assert os.path.exists(rosbag_launch_path), "rosbag launch file does not exist at path: {}".format(rosbag_launch_path) + + camera_calibration_dir = '/home/leonmkim/FISH_franka_ros_ws/src/contact_estimation_ros/fish_data_collection/scripts/camera_poses_L515/20240313-153328' +- dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, camera_calibration_dir, multicam=multicam, align_depth_to_color=align_depth_to_color) ++ dc_handler = RosbaggingHandler(base_dset_dir, experiment_dirname, rosbag_launch_path, demo_dirname, multicam=multicam, align_depth_to_color=align_depth_to_color) + + # rospy.spin() + diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e30b8f25539ba150b9a1eb06bb55373125f336f0 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dd247d1c4c04fa7cea7a03c76f9bcf0fac708360da8e1551f468b1b47fc66b9 +size 1171585 diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4425d998fb01594171bfd91b1cecddc8ba32d0aa --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35843717fd499551c4b0926cca736afc0945a08b7388201dffb14db7ec3fefa +size 1166652 diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa2d767c8503e3550834aae742bd058f03234bb7 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03a74d7b4bcab766fc29ee131b9e9ac693b435c897bde76d32747b19fb58257 +size 1184865 diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ba4f985c7115dec0a37f20b9d2516228ab7524e --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94417ff346f07f5fa23e9c53f6b0137c96c29dcb3d88023f1ed9c85959c133e3 +size 1210929 diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49141e9c64083e0f46d4c622006743c16480f4ed --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20ea9dbf78cd7cd19362e85b72b43f16e349294c086d44f5e3adc1fd73292d66 +size 1225749 diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/output.log b/205317/wandb/run-20241229_205323-2n31umej/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..27d7c3c4d493347244d2abff95eeece7e2ad5b68 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/output.log @@ -0,0 +1,225 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1735523610.488230]: resetting environment +[INFO] [1735523610.489337]: cleared current plan +[INFO] [1735523610.491255]: moving to home +[INFO] [1735523612.393259]: reached home +[INFO] [1735523612.394469]: reset action history +[INFO] [1735523615.152511]: environment reset +Starting episode 0 +[INFO] [1735523615.155349]: resetting environment +[INFO] [1735523615.156203]: cleared current plan +[INFO] [1735523615.156975]: moving to home +[INFO] [1735523616.159824]: reached home +[INFO] [1735523616.162780]: reset action history +[INFO] [1735523618.920455]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735523677.158728]: resetting environment +[INFO] [1735523677.160157]: cleared current plan +[INFO] [1735523677.161284]: moving to home +[INFO] [1735523678.163382]: reached home +[INFO] [1735523678.164495]: reset action history +[INFO] [1735523680.921619]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/episode_rosbags/episode_0_2024-12-29-20-54-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1355277.log +started roslaunch server http://158.130.50.37:34607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [1355542] +process[print_text-2]: started with pid [1355543] +process[rosbag_record-3]: started with pid [1355567] +started bagging! +For topic gripper_width: timestamp difference is -181985368 for nearest: 1735523701867554436 - target: 1735523702049539804 at idx 249 +[INFO] [1735523707.719766]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit[print_text-2] killing on exit +[INFO] [1735523708.445290]: Stored episode 1. +[INFO] [1735523708.445900]: Saving video... +[INFO] [1735523708.649843]: Video saved! +Starting episode 1 +[INFO] [1735523708.687548]: resetting environment +[INFO] [1735523708.687901]: cleared current plan +[INFO] [1735523708.688327]: moving to home +[INFO] [1735523712.294043]: reached home +[INFO] [1735523712.307056]: reset action history +[INFO] [1735523715.017036]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1735523725.239680]: resetting environment +[INFO] [1735523725.240496]: cleared current plan +[INFO] [1735523725.240901]: moving to home +[INFO] [1735523726.242374]: reached home +[INFO] [1735523726.242768]: reset action history +[INFO] [1735523728.999282]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/episode_rosbags/episode_1_2024-12-29-20-55-29.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1355277.log +started roslaunch server http://158.130.50.37:36565/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [1355671] +process[print_text-5]: started with pid [1355672] +process[rosbag_record-6]: started with pid [1355696] +started bagging! +For topic gripper_width: timestamp difference is -119824366 for nearest: 1735523737367585226 - target: 1735523737487409592 at idx 126 +For topic gripper_width: timestamp difference is -117102585 for nearest: 1735523740767553367 - target: 1735523740884655952 at idx 169 +For topic gripper_width: timestamp difference is -114201270 for nearest: 1735523744367552318 - target: 1735523744481753588 at idx 218 +[INFO] [1735523755.752178]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1735523756.515642]: Stored episode 2. +[INFO] [1735523756.515851]: Saving video... +[INFO] [1735523756.765216]: Video saved! +Starting episode 2 +[INFO] [1735523756.795257]: resetting environment +[INFO] [1735523756.808481]: cleared current plan +[INFO] [1735523756.813330]: moving to home +[INFO] [1735523760.325088]: reached home +[INFO] [1735523760.329300]: reset action history +[INFO] [1735523763.101227]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735523766.230032]: resetting environment +[INFO] [1735523766.235887]: cleared current plan +[INFO] [1735523766.242088]: moving to home +[INFO] [1735523767.249254]: reached home +[INFO] [1735523767.250467]: reset action history +[INFO] [1735523770.007904]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/episode_rosbags/episode_2_2024-12-29-20-56-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1355277.log +started roslaunch server http://158.130.50.37:46491/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [1355801] +process[print_text-8]: started with pid [1355802] +process[rosbag_record-9]: started with pid [1355803] +started bagging! +For topic gripper_width: timestamp difference is -121928547 for nearest: 1735523770567553427 - target: 1735523770689481974 at idx 12 +For topic gripper_width: timestamp difference is -117060210 for nearest: 1735523781967578862 - target: 1735523782084639072 at idx 175 +For topic gripper_width: timestamp difference is -113255168 for nearest: 1735523786767553901 - target: 1735523786880809069 at idx 240 +For topic gripper_width: timestamp difference is -109881919 for nearest: 1735523790967551906 - target: 1735523791077433825 at idx 249 +[INFO] [1735523796.754220]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1735523797.544603]: Stored episode 3. +[INFO] [1735523797.547160]: Saving video... +[INFO] [1735523797.840266]: Video saved! +Starting episode 3 +[INFO] [1735523797.913654]: resetting environment +[INFO] [1735523797.923165]: cleared current plan +[INFO] [1735523797.934853]: moving to home +[INFO] [1735523801.149350]: reached home +[INFO] [1735523801.153742]: reset action history +[INFO] [1735523804.016112]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735523806.043821]: resetting environment +[INFO] [1735523806.045080]: cleared current plan +[INFO] [1735523806.046980]: moving to home +[INFO] [1735523807.052008]: reached home +[INFO] [1735523807.060346]: reset action history +[INFO] [1735523809.887860]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/episode_rosbags/episode_3_2024-12-29-20-56-49.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1355277.log +started roslaunch server http://158.130.50.37:44079/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [1355952] +process[print_text-11]: started with pid [1355953] +process[rosbag_record-12]: started with pid [1355954] +started bagging! +For topic gripper_width: timestamp difference is -114542308 for nearest: 1735523826167607007 - target: 1735523826282149315 at idx 212 +For topic gripper_width: timestamp difference is -114427458 for nearest: 1735523826367560972 - target: 1735523826481988430 at idx 213 +For topic gripper_width: timestamp difference is -113633562 for nearest: 1735523827367551159 - target: 1735523827481184721 at idx 224 +For topic gripper_width: timestamp difference is -109286799 for nearest: 1735523832767577396 - target: 1735523832876864195 at idx 249 +[INFO] [1735523837.946021]: Storing episode... +[pub_text-10] killing on exit +[rosbag_record-12] killing on exit[print_text-11] killing on exit +[INFO] [1735523838.723363]: Stored episode 4. +[INFO] [1735523838.731264]: Saving video... +[INFO] [1735523839.282027]: Video saved! +Starting episode 4 +[INFO] [1735523839.351837]: resetting environment +[INFO] [1735523839.356024]: cleared current plan +[INFO] [1735523839.371458]: moving to home +[WARN] [1735523839.299034]: Plan exhausted +[WARN] [1735523839.338714]: Plan exhausted +[INFO] [1735523842.987996]: reached home +[INFO] [1735523843.005783]: reset action history +[INFO] [1735523845.794588]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1735523847.782817]: resetting environment +[INFO] [1735523847.787585]: cleared current plan +[INFO] [1735523847.792147]: moving to home +[INFO] [1735523848.802336]: reached home +[INFO] [1735523848.808479]: reset action history +[INFO] [1735523851.596875]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/episode_rosbags/episode_4_2024-12-29-20-57-31.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-1355277.log +started roslaunch server http://158.130.50.37:35151/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [1356084] +process[print_text-14]: started with pid [1356085] +process[rosbag_record-15]: started with pid [1356086] +started bagging! +[INFO] [1735523880.182312]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1735523880.957556]: Stored episode 5. +[INFO] [1735523880.967952]: Saving video... +[INFO] [1735523881.596726]: Video saved! +[INFO] [1735523881.664569]: resetting environment +[INFO] [1735523881.677550]: cleared current plan +[INFO] [1735523881.697741]: moving to home +[WARN] [1735523881.539209]: Plan exhausted +[WARN] [1735523881.582183]: Plan exhausted +[WARN] [1735523881.619458]: Plan exhausted +[WARN] [1735523881.658995]: Plan exhausted +[INFO] [1735523885.215016]: reached home +[INFO] [1735523885.231086]: reset action history +[INFO] [1735523888.142847]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! \ No newline at end of file diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt b/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json b/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..91427ebb82ae812b1ba487ff8f3afaf205a9f6ba --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-30T01:53:24.140645", + "startedAt": "2024-12-30T01:53:23.741358", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "23581857e509febdae7f0ac45cc9cf4f9081f85c" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3933.782375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4493.128, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.933, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 463.0247688293457 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json b/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c2a71f3063278553e71e89cb75ddc478eec4a65f --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "2dd247d1c4c04fa7cea7a03c76f9bcf0fac708360da8e1551f468b1b47fc66b9", "size": 1171585, "path": "media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4"}, "global_step": 660, "_timestamp": 1735523890.298451, "_runtime": 286.548122882843, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "f35843717fd499551c4b0926cca736afc0945a08b7388201dffb14db7ec3fefa", "size": 1166652, "path": "media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/2_eval": {"_type": "video-file", "sha256": "c03a74d7b4bcab766fc29ee131b9e9ac693b435c897bde76d32747b19fb58257", "size": 1184865, "path": "media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "94417ff346f07f5fa23e9c53f6b0137c96c29dcb3d88023f1ed9c85959c133e3", "size": 1210929, "path": "media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "20ea9dbf78cd7cd19362e85b72b43f16e349294c086d44f5e3adc1fd73292d66", "size": 1225749, "path": "media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4"}, "_wandb": {"runtime": 286}} \ No newline at end of file diff --git a/205317/wandb/run-20241229_205323-2n31umej/logs/debug-internal.log b/205317/wandb/run-20241229_205323-2n31umej/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..009547cb9cd45da872590b93cf6fcb48edcd6350 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/logs/debug-internal.log @@ -0,0 +1,749 @@ +2024-12-29 20:53:23,749 INFO StreamThr :1355438 [internal.py:wandb_internal():85] W&B internal server running at pid: 1355438, started at: 2024-12-29 20:53:23.749408 +2024-12-29 20:53:23,750 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status +2024-12-29 20:53:23,762 INFO WriterThread:1355438 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb +2024-12-29 20:53:23,762 DEBUG SenderThread:1355438 [sender.py:send():379] send: header +2024-12-29 20:53:23,770 DEBUG SenderThread:1355438 [sender.py:send():379] send: run +2024-12-29 20:53:23,990 INFO SenderThread:1355438 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files +2024-12-29 20:53:23,990 INFO SenderThread:1355438 [sender.py:_start_run_threads():1188] run started: 2n31umej with start time 1735523603.750328 +2024-12-29 20:53:24,005 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: check_version +2024-12-29 20:53:24,005 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: check_version +2024-12-29 20:53:24,122 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: run_start +2024-12-29 20:53:24,127 DEBUG HandlerThread:1355438 [system_info.py:__init__():26] System info init +2024-12-29 20:53:24,127 DEBUG HandlerThread:1355438 [system_info.py:__init__():41] System info init done +2024-12-29 20:53:24,127 INFO HandlerThread:1355438 [system_monitor.py:start():194] Starting system monitor +2024-12-29 20:53:24,127 INFO SystemMonitor:1355438 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-29 20:53:24,128 INFO HandlerThread:1355438 [system_monitor.py:probe():214] Collecting system info +2024-12-29 20:53:24,128 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started cpu monitoring +2024-12-29 20:53:24,129 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started disk monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started gpu monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started memory monitoring +2024-12-29 20:53:24,130 INFO SystemMonitor:1355438 [interfaces.py:start():188] Started network monitoring +2024-12-29 20:53:24,140 DEBUG HandlerThread:1355438 [system_info.py:probe():152] Probing system +2024-12-29 20:53:24,143 DEBUG HandlerThread:1355438 [system_info.py:_probe_git():137] Probing git +2024-12-29 20:53:24,149 DEBUG HandlerThread:1355438 [system_info.py:_probe_git():145] Probing git done +2024-12-29 20:53:24,149 DEBUG HandlerThread:1355438 [system_info.py:probe():200] Probing system done +2024-12-29 20:53:24,150 DEBUG HandlerThread:1355438 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-30T01:53:24.140645', 'startedAt': '2024-12-30T01:53:23.741358', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '23581857e509febdae7f0ac45cc9cf4f9081f85c'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3933.782375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4493.128, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.933, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 463.0247688293457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-29 20:53:24,150 INFO HandlerThread:1355438 [system_monitor.py:probe():224] Finished collecting system info +2024-12-29 20:53:24,150 INFO HandlerThread:1355438 [system_monitor.py:probe():227] Publishing system info +2024-12-29 20:53:24,150 DEBUG HandlerThread:1355438 [system_info.py:_save_code():44] Saving code +2024-12-29 20:53:24,151 DEBUG HandlerThread:1355438 [system_info.py:_save_code():67] Saving code done +2024-12-29 20:53:24,151 DEBUG HandlerThread:1355438 [system_info.py:_save_patches():84] Saving git patches +2024-12-29 20:53:24,221 DEBUG HandlerThread:1355438 [system_info.py:_save_patches():126] Saving git patches done +2024-12-29 20:53:24,222 INFO HandlerThread:1355438 [system_monitor.py:probe():229] Finished publishing system info +2024-12-29 20:53:24,225 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-29 20:53:24,225 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-29 20:53:24,459 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: python_packages +2024-12-29 20:53:24,459 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: python_packages +2024-12-29 20:53:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:24,462 INFO HandlerThread:1355438 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/tb" + save: true +} +control { + relay_id: "4ec06bb39130473e991b7d721c6b160e" +} +_info { + stream_id: "2n31umej" +} + +2024-12-29 20:53:24,462 DEBUG HandlerThread:1355438 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-29 20:53:24,473 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/hlo71ln1-wandb-metadata.json +2024-12-29 20:53:24,476 INFO wandb-upload_1:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/gq25rdy7-code/FISH/eval_robot.py +2024-12-29 20:53:24,493 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/cdppaczc-diff.patch +2024-12-29 20:53:24,565 DEBUG SenderThread:1355438 [sender.py:send():379] send: telemetry +2024-12-29 20:53:24,565 DEBUG SenderThread:1355438 [sender.py:send():379] send: tbrecord +2024-12-29 20:53:24,566 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:53:24,566 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 with policy live +2024-12-29 20:53:24,873 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/9djvv8q2-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:53:24,991 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH +2024-12-29 20:53:24,992 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code +2024-12-29 20:53:25,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:26,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:26,991 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:27,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:28,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:29,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:29,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:30,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:30,488 DEBUG SenderThread:1355438 [sender.py:send():379] send: config +2024-12-29 20:53:31,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:32,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:32,993 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:33,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:34,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:34,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:34,993 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:35,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:36,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:36,994 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:37,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:38,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:38,995 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:39,459 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:39,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:39,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:39,622 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:40,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:40,995 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:53:41,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:42,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:43,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:44,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:45,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:45,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:46,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:47,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:48,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:49,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:50,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:51,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:51,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:52,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:53,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:54,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:53:54,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:53:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:55,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:56,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:56,527 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:53:56,999 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml +2024-12-29 20:53:57,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:58,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:53:59,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:00,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:01,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:02,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:02,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:03,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:04,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:05,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:06,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:07,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:07,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:08,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:09,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:09,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:09,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:10,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:11,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:12,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:12,558 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:13,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:14,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:15,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:16,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:17,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:18,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:18,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:19,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:20,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:21,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:22,468 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:23,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:23,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:24,131 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-29 20:54:24,131 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:54:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:24,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:25,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:26,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:27,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:28,469 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:28,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:29,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:30,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:31,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:32,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:33,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:34,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:34,470 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:35,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:36,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:37,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:38,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:39,011 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:39,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:39,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:39,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:39,548 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:40,471 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:41,012 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:41,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:42,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:43,013 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:54:43,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:44,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:45,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:45,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:46,472 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:47,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:48,473 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:49,473 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:50,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:50,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:51,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:52,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:53,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:54,132 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:54:54,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:54:54,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:54:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:55,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:55,572 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:54:56,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:57,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:58,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:54:59,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:00,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:01,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:01,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:02,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:03,018 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:03,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:04,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:05,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:06,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:06,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:07,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:08,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:08,697 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:55:08,698 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 with policy now +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:55:08,698 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:55:08,699 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:55:08,976 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/j79b7238-media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,020 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media +2024-12-29 20:55:09,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos +2024-12-29 20:55:09,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:55:09,330 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/qjvpmhlf-media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:55:09,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:09,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:09,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:10,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:11,021 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:11,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:11,599 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:12,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:13,488 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:14,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:15,022 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:15,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:16,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:17,023 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:17,061 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:17,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:18,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:19,489 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:20,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:21,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:22,463 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:22,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:23,490 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:24,132 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:55:24,460 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:24,460 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:24,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:25,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:26,026 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:55:26,435 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/7e1on89c-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:55:26,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:27,026 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:27,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:28,285 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:28,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:29,027 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:29,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:30,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:31,028 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:31,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:32,491 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:33,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:33,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:34,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:35,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:36,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:37,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:38,492 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:38,585 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:39,030 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:39,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:39,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:39,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:40,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:41,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:42,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:43,032 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:43,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:43,989 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:44,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:45,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:46,494 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:47,033 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:47,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:48,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:49,464 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:49,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:50,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:51,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:52,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:53,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:54,133 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:55:54,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:55:54,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:55:54,501 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:54,596 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:55:55,495 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:56,496 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:56,793 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:55:56,794 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 with policy now +2024-12-29 20:55:56,794 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:55:56,795 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:55:56,795 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:55:56,795 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:55:57,085 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/vqbofts6-media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,378 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/px30hwfz-media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:55:57,496 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:58,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:55:59,037 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:55:59,497 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:00,326 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:00,498 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:01,500 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:02,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:03,039 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:03,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:04,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:05,039 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:05,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:05,503 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:06,502 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:06,952 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:06,952 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:06,952 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:07,040 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:07,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:08,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:09,041 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:09,461 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:09,461 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:09,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:10,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:10,743 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:11,041 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:11,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:12,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:13,042 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:13,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:14,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:15,505 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:16,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:16,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:17,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:18,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:19,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:20,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:21,465 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:21,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:22,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:23,045 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:23,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:24,133 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:56:24,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:24,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:24,506 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:25,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:26,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:26,602 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:27,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:28,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:29,047 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:29,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:30,507 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:31,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:32,185 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:32,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:33,049 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:33,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:34,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:35,508 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:36,509 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:37,511 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:37,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:37,891 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:56:37,891 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 with policy now +2024-12-29 20:56:37,897 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:56:37,898 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:37,898 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:37,898 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:56:38,125 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/80bprm7k-media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,413 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/aun1d2pr-media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:56:38,520 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:39,051 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:39,480 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:39,480 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:39,521 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:40,517 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:41,520 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:42,534 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:43,052 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:43,197 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:43,535 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:44,532 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:45,053 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:45,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:46,053 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:46,307 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/tyopdo6f-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:56:46,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:46,784 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:56:46,784 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:56:46,784 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:56:47,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:56:47,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:47,540 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:48,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:49,054 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:49,093 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:49,541 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:50,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:51,055 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:51,541 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:52,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:53,056 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:56:53,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:54,134 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:56:54,134 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:54,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:56:54,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:56:54,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:55,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:56,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:57,542 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:58,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:56:59,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:56:59,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:00,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:01,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:02,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:03,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:04,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:04,543 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:05,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:06,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:07,577 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:08,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:09,061 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:09,462 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:09,462 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:09,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:09,611 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:10,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:11,544 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:12,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:13,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:14,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:15,063 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:15,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:15,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:16,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:17,545 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:18,549 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:19,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:19,344 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:57:19,344 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 with policy now +2024-12-29 20:57:19,347 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:57:19,347 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:57:19,347 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:57:19,347 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:57:19,550 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:19,568 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/qnlxi08l-media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:57:20,065 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:57:20,366 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/l9ny6g1n-media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:57:20,474 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:20,551 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:21,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:21,552 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:22,066 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:22,558 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:23,556 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:24,134 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:57:24,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:24,476 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:24,556 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:25,067 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:25,551 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:25,634 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:26,554 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:27,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:27,554 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:27,831 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:57:27,831 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:57:27,832 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:57:28,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:57:28,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:57:28,562 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:29,068 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:29,560 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:30,557 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:31,069 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:31,477 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:31,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:32,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:33,070 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:33,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:34,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:35,070 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:57:35,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:36,564 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:37,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:37,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:38,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:39,466 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:39,467 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:39,566 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:40,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:41,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:42,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:42,616 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:43,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:44,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:45,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:46,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:47,567 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:48,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:48,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:49,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:50,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:51,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:52,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:53,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:54,135 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:57:54,136 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:54,467 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:57:54,467 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:57:54,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:55,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:56,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:57,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:58,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:57:59,476 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:57:59,568 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:00,569 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:01,079 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:01,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:01,655 DEBUG SenderThread:1355438 [sender.py:send():379] send: files +2024-12-29 20:58:01,655 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 with policy now +2024-12-29 20:58:01,658 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: partial_history +2024-12-29 20:58:01,658 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:01,659 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:01,659 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:01,912 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/hzibu7kv-media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval +2024-12-29 20:58:02,358 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/m967u2jl-media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:02,605 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:03,080 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:03,606 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:04,081 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:04,483 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:04,600 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:05,610 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:06,632 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:07,081 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:07,620 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:08,625 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:09,082 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:09,475 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: stop_status +2024-12-29 20:58:09,475 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: stop_status +2024-12-29 20:58:09,573 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:09,614 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:10,337 DEBUG SenderThread:1355438 [sender.py:send():379] send: exit +2024-12-29 20:58:10,337 INFO SenderThread:1355438 [sender.py:send_exit():586] handling exit code: 0 +2024-12-29 20:58:10,337 INFO SenderThread:1355438 [sender.py:send_exit():588] handling runtime: 286 +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:send_exit():594] send defer +2024-12-29 20:58:10,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,338 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-29 20:58:10,338 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-29 20:58:10,338 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 1 +2024-12-29 20:58:10,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,338 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-29 20:58:10,339 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,339 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-29 20:58:10,339 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 2 +2024-12-29 20:58:10,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [system_monitor.py:finish():203] Stopping system monitor +2024-12-29 20:58:10,339 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined cpu monitor +2024-12-29 20:58:10,339 DEBUG SystemMonitor:1355438 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-29 20:58:10,339 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined disk monitor +2024-12-29 20:58:10,388 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined gpu monitor +2024-12-29 20:58:10,389 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined memory monitor +2024-12-29 20:58:10,389 INFO HandlerThread:1355438 [interfaces.py:finish():200] Joined network monitor +2024-12-29 20:58:10,389 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,389 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-29 20:58:10,389 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 3 +2024-12-29 20:58:10,389 DEBUG SenderThread:1355438 [sender.py:send():379] send: stats +2024-12-29 20:58:10,389 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,390 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-29 20:58:10,390 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:10,390 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-29 20:58:10,390 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 4 +2024-12-29 20:58:10,390 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:10,390 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:11,083 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:11,336 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /tmp/tmpigpyerh5wandb/q3m3qog2-events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:16,392 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,392 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: keepalive +2024-12-29 20:58:16,392 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-29 20:58:16,393 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 5 +2024-12-29 20:58:16,393 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:16,393 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,393 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:16,393 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-29 20:58:16,393 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send():379] send: history +2024-12-29 20:58:16,394 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: summary_record +2024-12-29 20:58:16,397 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send():379] send: summary +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 6 +2024-12-29 20:58:16,398 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,398 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-29 20:58:16,398 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,398 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-29 20:58:16,399 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 7 +2024-12-29 20:58:16,399 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: status_report +2024-12-29 20:58:16,399 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:16,399 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-29 20:58:16,399 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:16,399 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-29 20:58:17,085 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:17,338 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:17,357 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 8 +2024-12-29 20:58:17,357 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:17,357 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:17,357 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-29 20:58:17,358 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:17,358 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-29 20:58:17,358 INFO SenderThread:1355438 [job_builder.py:build():440] Attempting to build job artifact +2024-12-29 20:58:17,360 INFO SenderThread:1355438 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-29 20:58:17,376 INFO SenderThread:1355438 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-29 20:58:17,387 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 9 +2024-12-29 20:58:17,387 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:17,387 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-29 20:58:17,387 DEBUG SenderThread:1355438 [sender.py:send():379] send: artifact +2024-12-29 20:58:18,021 INFO wandb-upload_1:1355438 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0szy62ux +2024-12-29 20:58:18,085 INFO Thread-12 :1355438 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:18,124 INFO wandb-upload_0:1355438 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgsnxelt6 +2024-12-29 20:58:18,255 INFO wandb-upload_2:1355438 [upload_job.py:push():88] Uploaded file /tmp/tmpf0ewpl8h/wandb-job.json +2024-12-29 20:58:18,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxMjA1NjUzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MzQyNDMyMw==', 'versionIndex': 385}}} +2024-12-29 20:58:18,817 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-29 20:58:18,817 INFO SenderThread:1355438 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml config.yaml +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json wandb-summary.json +2024-12-29 20:58:19,086 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 events.out.tfevents.1735523604.leonmkim-ROG-Strix-G15CS-G15CS.1355277.0 +2024-12-29 20:58:19,088 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt requirements.txt +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-metadata.json wandb-metadata.json +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log output.log +2024-12-29 20:58:19,089 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/diff.patch diff.patch +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 media/videos/eval/1_eval_1_f35843717fd499551c4b.mp4 +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 media/videos/eval/0_eval_0_2dd247d1c4c04fa7cea7.mp4 +2024-12-29 20:58:19,092 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 media/videos/eval/2_eval_2_c03a74d7b4bcab766fc2.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 media/videos/eval/3_eval_3_94417ff346f07f5fa23e.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 media/videos/eval/4_eval_4_20ea9dbf78cd7cd19362.mp4 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 10 +2024-12-29 20:58:19,093 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,093 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,093 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-29 20:58:19,093 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-29 20:58:19,093 INFO SenderThread:1355438 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 20:58:19,264 INFO wandb-upload_1:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/config.yaml +2024-12-29 20:58:19,266 INFO wandb-upload_2:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/wandb-summary.json +2024-12-29 20:58:19,300 INFO wandb-upload_0:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/requirements.txt +2024-12-29 20:58:19,330 INFO wandb-upload_3:1355438 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/files/output.log +2024-12-29 20:58:19,339 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,340 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,531 INFO Thread-11 (_thread_body):1355438 [sender.py:transition_state():614] send defer: 11 +2024-12-29 20:58:19,531 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,531 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-29 20:58:19,531 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,531 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-29 20:58:19,531 INFO SenderThread:1355438 [file_pusher.py:join():175] waiting for file pusher +2024-12-29 20:58:19,532 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 12 +2024-12-29 20:58:19,532 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,532 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-29 20:58:19,532 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,532 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-29 20:58:19,533 INFO SenderThread:1355438 [file_stream.py:finish():601] file stream finish called +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [file_stream.py:finish():605] file stream finish is done +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 13 +2024-12-29 20:58:19,591 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,591 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-29 20:58:19,591 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-29 20:58:19,591 INFO SenderThread:1355438 [sender.py:transition_state():614] send defer: 14 +2024-12-29 20:58:19,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: defer +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send():379] send: final +2024-12-29 20:58:19,592 INFO HandlerThread:1355438 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send():379] send: footer +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: defer +2024-12-29 20:58:19,592 INFO SenderThread:1355438 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-29 20:58:19,592 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,592 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,593 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: server_info +2024-12-29 20:58:19,593 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: server_info +2024-12-29 20:58:19,593 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-29 20:58:19,594 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: get_summary +2024-12-29 20:58:19,594 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-29 20:58:19,595 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-29 20:58:19,682 DEBUG SenderThread:1355438 [sender.py:send_request():406] send_request: poll_exit +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-29 20:58:19,682 INFO MainThread:1355438 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-29 20:58:19,683 DEBUG HandlerThread:1355438 [handler.py:handle_request():158] handle_request: shutdown +2024-12-29 20:58:19,683 INFO HandlerThread:1355438 [handler.py:finish():882] shutting down handler +2024-12-29 20:58:20,594 INFO WriterThread:1355438 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [sender.py:finish():1615] shutting down sender +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [file_pusher.py:finish():169] shutting down file pusher +2024-12-29 20:58:20,682 INFO SenderThread:1355438 [file_pusher.py:join():175] waiting for file pusher diff --git a/205317/wandb/run-20241229_205323-2n31umej/logs/debug.log b/205317/wandb/run-20241229_205323-2n31umej/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1e022ee97b2e3d60795a66ad04615295dfde2c73 --- /dev/null +++ b/205317/wandb/run-20241229_205323-2n31umej/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Configure stats pid to 1355277 +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/settings +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/logs/debug.log +2024-12-29 20:53:23,743 INFO MainThread:1355277 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/wandb/run-20241229_205323-2n31umej/logs/debug-internal.log +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():569] calling init triggers +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 5, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1045_0', 'wandb_notes': '1045_0_', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['greece_twodim_nominal', 'greece_twodim_recovery'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 0, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317'} +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():619] starting backend +2024-12-29 20:53:23,744 INFO MainThread:1355277 [wandb_init.py:init():623] setting up manager +2024-12-29 20:53:23,748 INFO MainThread:1355277 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-29 20:53:23,750 INFO MainThread:1355277 [wandb_init.py:init():631] backend started and connected +2024-12-29 20:53:23,761 INFO MainThread:1355277 [wandb_init.py:init():720] updated telemetry +2024-12-29 20:53:23,767 INFO MainThread:1355277 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-29 20:53:24,005 INFO MainThread:1355277 [wandb_run.py:_on_init():2435] communicating current version +2024-12-29 20:53:24,117 INFO MainThread:1355277 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-29 20:53:24,117 INFO MainThread:1355277 [wandb_init.py:init():804] starting run threads in backend +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_console_start():2413] atexit reg +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-29 20:53:24,460 INFO MainThread:1355277 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-29 20:53:24,461 INFO MainThread:1355277 [wandb_init.py:init():847] run started, returning control to user process +2024-12-29 20:53:24,462 INFO MainThread:1355277 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1045_0/205317/tb, True +2024-12-29 20:53:30,487 INFO MainThread:1355277 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2024-12-29 20:58:20,684 WARNING MsgRouterThr:1355277 [router.py:message_loop():77] message_loop has been closed diff --git a/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb b/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3bf236fd042ffd7dd9ccfdc80b155ca550625f1b Binary files /dev/null and b/205317/wandb/run-20241229_205323-2n31umej/run-2n31umej.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c15ae29d90ee60a289d1731913ddff30c579f4ef --- /dev/null +++ b/config.yaml @@ -0,0 +1,548 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 5 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1045_0' +wandb_notes: '1045_0_' +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- greece_twodim_nominal +- greece_twodim_recovery +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 0 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a134e7e2aafbcf79b90ff8e066d23fa1feedee7 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a969ffbdc5d4f82b42645ee428ed42ce53516504ab5469a4a8b03ada3af4386d +size 910064728 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b08cffe08ba59d61ab098a487adb91d5608a86f2 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c57f96ed3083c018508a64fca7c89aa2aa61eb201819d2b71e84551952c8413f +size 910064728 diff --git a/snapshot_12999.pt b/snapshot_12999.pt new file mode 100644 index 0000000000000000000000000000000000000000..d820b81b285fdf816b76a8b023511a55afda0abd --- /dev/null +++ b/snapshot_12999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83031bd9a71dace532377d29868da3976cae276f497779868fb835c55d98a62a +size 910064728 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb1b1a15fe758871965747a64eb06423faab9aa7 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6dc9172d30a9852821d1a0d74fc847537782edceca8e6d76dd2af5224f4816e +size 910064728 diff --git a/snapshot_13749.pt b/snapshot_13749.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9cc410dc73c9616c7e603f59654f0541e3c3a64 --- /dev/null +++ b/snapshot_13749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7996539c7434450c85aff854080b92623b77d41d79d749ee44d6c640bb4f8272 +size 910064728 diff --git a/snapshot_14249.pt b/snapshot_14249.pt new file mode 100644 index 0000000000000000000000000000000000000000..d44059ae57c79a5eae9b37d59e3db2d6f9ac6a39 --- /dev/null +++ b/snapshot_14249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab4379747afce45b086c41766cbfad4a2fc01914c5fcf4789e789047ae29390 +size 910064728 diff --git a/snapshot_14499.pt b/snapshot_14499.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f62808f2f05787ddd840fc4631dc68ca0eb9c08 --- /dev/null +++ b/snapshot_14499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d65a1143797b7374af4c70c062b8f9650e55f281b77cf56eab4cd02b1cb036 +size 910064728 diff --git a/snapshot_14749.pt b/snapshot_14749.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf4d06018244b8ec12998788384a3ca6a0a2590d --- /dev/null +++ b/snapshot_14749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dfad5eb4fe302b08d920de8a868b0fb65fb9d699b8a385223676da47f1e7215 +size 910064728 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..feaa243919547116cdc406c63af85c5375833a19 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dffe18f6f67d61277cb2dc2b7f5aa02f4524736ecc48e41101f376aa538df2f +size 910064728 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..898b903dc2047e460f95bc38dbbdfc6bacd0cbca --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e274c728aa386f9de3803ba1865e8b4294867fddb6fef7fd1b638abaa54ac2 +size 910064728 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..921a18ab7f6117fe6fa63c0a7de1201ea978bb45 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a481522777c050c9af17e1bed905e8e20eb69061c4eacc65031c7b367aa9d060 +size 910064728 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b830a005c127662856a73d9e69a1f7da6141f04 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b166de32436acf1d74de901c4b59460191c64d0d9f4741858d505f5cc6309e +size 910064728 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d5e601946a4bcb7d5eb7f8e48ee0318521ef33a --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ae78bb8f5804722d0e6e89a96eabcf1fa1cbd269d0eb98eee0d1dcb3786c47d +size 910064728 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..feaa243919547116cdc406c63af85c5375833a19 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dffe18f6f67d61277cb2dc2b7f5aa02f4524736ecc48e41101f376aa538df2f +size 910064728