diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a9853351f10ebfa3dd10905c9420a75e5fce4570 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 filter=lfs diff=lfs merge=lfs -text +191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 filter=lfs diff=lfs merge=lfs -text +191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 filter=lfs diff=lfs merge=lfs -text +191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 filter=lfs diff=lfs merge=lfs -text +191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 filter=lfs diff=lfs merge=lfs -text +191334/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +191334/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +191334/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +191334/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +191334/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +191334/episode_rosbags/episode_3_2024-12-11-19-16-37.bag filter=lfs diff=lfs merge=lfs -text +191334/episode_rosbags/episode_4_2024-12-11-19-17-19.bag filter=lfs diff=lfs merge=lfs -text +191334/episode_rosbags/episode_0_2024-12-11-19-14-01.bag filter=lfs diff=lfs merge=lfs -text +191334/episode_rosbags/episode_1_2024-12-11-19-14-40.bag filter=lfs diff=lfs merge=lfs -text +191334/episode_rosbags/episode_2_2024-12-11-19-15-49.bag filter=lfs diff=lfs merge=lfs -text diff --git a/191332/.hydra/config.yaml b/191332/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..817d92fd0c2d9899707311060302a486e47d6985 --- /dev/null +++ b/191332/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '266406_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/191332/.hydra/hydra.yaml b/191332/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7972c131e4e598e4f812882fb22eacf60f5ea4f9 --- /dev/null +++ b/191332/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191332 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/191332/.hydra/overrides.yaml b/191332/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/191332/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/191332/eval_policy.log b/191332/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..74c1c8042c22bcfba686c5282a2af9444c28302f --- /dev/null +++ b/191332/eval_policy.log @@ -0,0 +1,15 @@ +[2024-12-11 19:13:32,823][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 19:13:32,826][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2024-12-11 19:13:36,207][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/191334/.hydra/config.yaml b/191334/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..817d92fd0c2d9899707311060302a486e47d6985 --- /dev/null +++ b/191334/.hydra/config.yaml @@ -0,0 +1,349 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 192_240x320_multiobj_twodim_recovery_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '266406_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/191334/.hydra/hydra.yaml b/191334/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49fa2aeed6e01b051f65cb6d246aa5da05146a8e --- /dev/null +++ b/191334/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${experiment_dir}/${now:%H%M%S} + sweep: + dir: ${experiment_dir}/${now:%H%M%S} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${experiment_dir}/${now:%H%M%S}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/191334/.hydra/overrides.yaml b/191334/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/191334/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/191334/episode_rosbags/aligned_depth_to_color_K.npy b/191334/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/191334/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/191334/episode_rosbags/cam_tf_world.npy b/191334/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/191334/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/191334/episode_rosbags/color_K.npy b/191334/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/191334/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/191334/episode_rosbags/depth_K.npy b/191334/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/191334/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/191334/episode_rosbags/episode_0_2024-12-11-19-14-01.bag b/191334/episode_rosbags/episode_0_2024-12-11-19-14-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..cff998523e2505c3cfc22a9dd538d0a615bfa631 --- /dev/null +++ b/191334/episode_rosbags/episode_0_2024-12-11-19-14-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a155461da1a2dfae39a56f4efc0d7b26ed5b94e3bb376701ac85a5e2cf3ce0e +size 1358855173 diff --git a/191334/episode_rosbags/episode_1_2024-12-11-19-14-40.bag b/191334/episode_rosbags/episode_1_2024-12-11-19-14-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..7c9ebd360527a54c492c7694abfd669022042d63 --- /dev/null +++ b/191334/episode_rosbags/episode_1_2024-12-11-19-14-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54d2ad33a56c44359af30095a50116094bce2df633bba604d9b39bdcc75457b5 +size 1365117617 diff --git a/191334/episode_rosbags/episode_2_2024-12-11-19-15-49.bag b/191334/episode_rosbags/episode_2_2024-12-11-19-15-49.bag new file mode 100644 index 0000000000000000000000000000000000000000..7abfb52ef2c92add32b9168c45a9f77ce96999e8 --- /dev/null +++ b/191334/episode_rosbags/episode_2_2024-12-11-19-15-49.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7c1730b0d0f0c7bffa9171211a845fb18546f7817caf381ae10ac04be953443 +size 1363946609 diff --git a/191334/episode_rosbags/episode_3_2024-12-11-19-16-37.bag b/191334/episode_rosbags/episode_3_2024-12-11-19-16-37.bag new file mode 100644 index 0000000000000000000000000000000000000000..65e4ee960275c3ecf405f96a8980f1adc491cccf --- /dev/null +++ b/191334/episode_rosbags/episode_3_2024-12-11-19-16-37.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc9e73a2ad25d7637db6b335bf6927ff08607a15fc0e7a52ae230dfde246d20a +size 1365532312 diff --git a/191334/episode_rosbags/episode_4_2024-12-11-19-17-19.bag b/191334/episode_rosbags/episode_4_2024-12-11-19-17-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..6f93c201e530e45e380338108a034fc5046923b5 --- /dev/null +++ b/191334/episode_rosbags/episode_4_2024-12-11-19-17-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a5997dbddd47c04d669922d6cbd1718e658edaa1497ac0d79c8a0b84787b77a +size 1361871865 diff --git a/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/.zgroup b/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/.zgroup new file mode 100644 index 0000000000000000000000000000000000000000..3b7daf227c1687f28bc23b69f183e27ce9a475c1 --- /dev/null +++ b/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/data/.zgroup b/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/data/.zgroup new file mode 100644 index 0000000000000000000000000000000000000000..3b7daf227c1687f28bc23b69f183e27ce9a475c1 --- /dev/null +++ b/191334/episode_zarrs/episode_0_2024-12-11-19-14-01.zarr/data/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format": 2 +} \ No newline at end of file diff --git a/191334/eval_robot.log b/191334/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..ba80374bcb148581af002f3b08ed25f4796a9b7c --- /dev/null +++ b/191334/eval_robot.log @@ -0,0 +1,12 @@ +[2024-12-11 19:13:34,754][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:503: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2024-12-11 19:13:34,757][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:439: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/191334/eval_video/0_eval.mp4 b/191334/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3734f776458717055cd3bb6c4fb63a4a4387c8a3 --- /dev/null +++ b/191334/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da136561a2ad42749fdae27d6475d37bd6904f003bb617bd102700e75835318 +size 1151012 diff --git a/191334/eval_video/1_eval.mp4 b/191334/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5433e8db927982af4f062c2ac2d023f00d001a57 --- /dev/null +++ b/191334/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a45ace652f05385ee7562019925e1bc1406b8fdc00f0c2a8c0f3a6261051e5f +size 1151665 diff --git a/191334/eval_video/2_eval.mp4 b/191334/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..390dec095718c79514dc3e2093a2d20721865b11 --- /dev/null +++ b/191334/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9cccf7ec7cae45b2fa152dff13e74643ef89365bb489c5d108e4a154438176e +size 1239572 diff --git a/191334/eval_video/3_eval.mp4 b/191334/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45a1449b181c5e1d83cfdbb86b687a5eed1ee829 --- /dev/null +++ b/191334/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334d1f02994a0fcdeda5419961508a7e674f6bbf2f7c4b38d27d336ee27676ec +size 1243778 diff --git a/191334/eval_video/4_eval.mp4 b/191334/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f81363f0242e3532d1c60fb3aaff7e879fbd3dec --- /dev/null +++ b/191334/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35283caf4103c3829cd1d42553a2a9aa55f66bad96d4d9acf1eb68d56d1ef317 +size 1268903 diff --git a/191334/tb/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 b/191334/tb/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 new file mode 100644 index 0000000000000000000000000000000000000000..3e5ee2b884bdc4ae95a70e795442092c394059a8 --- /dev/null +++ b/191334/tb/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d1854ed1efc7c51777bbefd9b4a35f2b95e5c16066781df5b5b2982a3d43b2b +size 1103 diff --git a/191334/wandb/debug-internal.log b/191334/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d7c220c3fc4b8b136df74bc6bbfb1f4b5bb50b4a --- /dev/null +++ b/191334/wandb/debug-internal.log @@ -0,0 +1,755 @@ +2024-12-11 19:13:40,902 INFO StreamThr :773446 [internal.py:wandb_internal():85] W&B internal server running at pid: 773446, started at: 2024-12-11 19:13:40.902318 +2024-12-11 19:13:40,903 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status +2024-12-11 19:13:40,918 INFO WriterThread:773446 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb +2024-12-11 19:13:40,918 DEBUG SenderThread:773446 [sender.py:send():379] send: header +2024-12-11 19:13:40,926 DEBUG SenderThread:773446 [sender.py:send():379] send: run +2024-12-11 19:13:41,175 INFO SenderThread:773446 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files +2024-12-11 19:13:41,175 INFO SenderThread:773446 [sender.py:_start_run_threads():1188] run started: l07wa4hd with start time 1733962420.904478 +2024-12-11 19:13:41,191 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 19:13:41,192 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: check_version +2024-12-11 19:13:41,268 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 19:13:41,273 DEBUG HandlerThread:773446 [system_info.py:__init__():26] System info init +2024-12-11 19:13:41,273 DEBUG HandlerThread:773446 [system_info.py:__init__():41] System info init done +2024-12-11 19:13:41,273 INFO HandlerThread:773446 [system_monitor.py:start():194] Starting system monitor +2024-12-11 19:13:41,273 INFO SystemMonitor:773446 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 19:13:41,273 INFO HandlerThread:773446 [system_monitor.py:probe():214] Collecting system info +2024-12-11 19:13:41,273 INFO SystemMonitor:773446 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 19:13:41,274 INFO SystemMonitor:773446 [interfaces.py:start():188] Started disk monitoring +2024-12-11 19:13:41,275 INFO SystemMonitor:773446 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 19:13:41,275 INFO SystemMonitor:773446 [interfaces.py:start():188] Started memory monitoring +2024-12-11 19:13:41,276 INFO SystemMonitor:773446 [interfaces.py:start():188] Started network monitoring +2024-12-11 19:13:41,286 DEBUG HandlerThread:773446 [system_info.py:probe():152] Probing system +2024-12-11 19:13:41,288 DEBUG HandlerThread:773446 [system_info.py:_probe_git():137] Probing git +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:_probe_git():145] Probing git done +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:probe():200] Probing system done +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-12T00:13:41.286723', 'startedAt': '2024-12-12T00:13:40.893574', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.654375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.045, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 705.1167831420898}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 19:13:41,293 INFO HandlerThread:773446 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 19:13:41,293 INFO HandlerThread:773446 [system_monitor.py:probe():227] Publishing system info +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:_save_code():44] Saving code +2024-12-11 19:13:41,294 DEBUG HandlerThread:773446 [system_info.py:_save_code():67] Saving code done +2024-12-11 19:13:41,294 DEBUG HandlerThread:773446 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 19:13:41,362 DEBUG HandlerThread:773446 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 19:13:41,363 INFO HandlerThread:773446 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 19:13:41,366 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:13:41,366 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 19:13:41,366 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 19:13:41,367 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 19:13:41,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 19:13:41,616 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: python_packages +2024-12-11 19:13:41,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:13:41,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:13:41,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:41,619 INFO HandlerThread:773446 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/tb" + save: true +} +control { + relay_id: "85246c7cda7d49f9afbefde503c903d6" +} +_info { + stream_id: "l07wa4hd" +} + +2024-12-11 19:13:41,619 DEBUG HandlerThread:773446 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 19:13:41,639 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/r8wlspo2-wandb-metadata.json +2024-12-11 19:13:41,640 INFO wandb-upload_2:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/azg7zrgd-diff.patch +2024-12-11 19:13:41,683 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/3fipiwh0-code/FISH/eval_robot.py +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: telemetry +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: tbrecord +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:13:41,705 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 with policy live +2024-12-11 19:13:41,892 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/77gm1cue-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:13:42,177 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:42,177 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH +2024-12-11 19:13:42,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:43,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:44,176 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:44,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:45,501 DEBUG SenderThread:773446 [sender.py:send():379] send: config +2024-12-11 19:13:45,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:46,507 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:46,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:47,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:48,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:48,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:49,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:50,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:51,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:51,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:52,179 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:52,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:53,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:54,179 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:54,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:55,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:56,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:13:56,616 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:13:56,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:56,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:57,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:58,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:59,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:00,181 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:00,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:01,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:02,182 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:02,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:02,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:03,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:04,182 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:04,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:05,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:06,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:07,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:08,011 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:08,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:09,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:10,184 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:10,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:11,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:11,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:11,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:12,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:13,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:13,745 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:14,185 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml +2024-12-11 19:14:14,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:15,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:16,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:17,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:18,186 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:18,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:19,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:19,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:20,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:21,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:22,187 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:22,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:23,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:24,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:24,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:25,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:26,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:26,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:26,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:27,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:28,623 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:29,326 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:14:29,327 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 with policy now +2024-12-11 19:14:29,327 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:14:29,328 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:14:29,328 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:14:29,329 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:14:29,595 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/p9oiypsh-media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:29,623 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media +2024-12-11 19:14:30,332 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:30,478 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/otzzej9i-media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:30,625 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:31,626 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:32,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:32,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:33,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:34,191 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:34,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:35,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:35,886 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:36,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:36,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:37,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:38,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:14:38,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:38,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:39,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:40,193 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:40,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:40,939 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:41,279 DEBUG SystemMonitor:773446 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 19:14:41,280 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:14:41,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:41,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:41,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:42,194 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:42,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:43,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:44,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:45,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:46,558 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:46,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:47,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:48,196 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:48,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:49,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:50,196 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:50,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:51,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:51,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:52,197 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:52,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:53,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:54,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:55,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:56,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:56,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:56,658 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:56,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:57,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:58,199 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:58,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:59,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:00,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:01,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:02,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:02,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:03,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:04,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:05,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:06,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:07,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:08,326 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:08,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:09,596 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:15:09,596 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 with policy now +2024-12-11 19:15:09,597 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:15:09,598 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:15:09,598 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:15:09,600 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:15:09,636 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:09,849 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/9qh203tq-media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:15:10,471 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/4acat9m8-media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,638 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:11,280 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:15:11,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:11,622 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:11,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:12,204 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:12,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:13,550 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:13,639 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:14,204 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:14,640 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:15,643 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:16,205 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:16,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:17,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:18,205 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:18,626 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:18,650 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:19,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:20,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:21,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:22,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:23,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:24,624 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:24,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:25,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:26,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:26,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:26,661 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:27,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:28,650 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:29,651 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:29,733 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:30,655 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:31,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:32,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:33,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:34,655 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:35,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:35,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:36,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:37,658 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:38,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:39,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:40,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:40,656 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:41,281 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:15:41,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:41,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:41,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:42,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:43,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:44,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:45,664 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:45,739 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:46,215 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:15:46,451 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/7wdb8k1w-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:15:46,670 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:47,040 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:15:47,040 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:15:47,040 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:15:47,215 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:15:47,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:48,216 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:48,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:49,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:50,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:50,978 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:51,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:52,217 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:52,670 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:53,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:54,218 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:54,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:55,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:56,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:56,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:56,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:56,737 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:57,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:58,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:59,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:00,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:01,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:02,087 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:02,220 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:02,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:03,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:04,221 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:04,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:05,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:06,222 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:06,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:07,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:07,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:08,222 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:08,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:09,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:10,223 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:10,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:11,281 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:16:11,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:11,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:11,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:12,224 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:12,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:12,691 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:13,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:14,224 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:14,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:15,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:16,225 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:16,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:17,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:18,226 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:18,286 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:18,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:19,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:20,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:20,673 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:20,921 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:16:20,921 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 with policy now +2024-12-11 19:16:20,922 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:16:20,922 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:16:20,922 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:16:20,923 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:16:21,212 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/wobof1o8-media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:16:21,502 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/a144xsb1-media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,676 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:22,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:22,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:23,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:23,693 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:24,675 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:25,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:26,229 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:26,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:26,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:26,672 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:27,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:28,229 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:28,681 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:28,689 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:29,688 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:30,689 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:31,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:32,682 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:33,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:33,879 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:16:33,882 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:16:33,882 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:33,883 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:16:34,231 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:16:34,231 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:16:34,232 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:34,683 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:35,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:36,232 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:36,681 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:37,684 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:38,233 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:38,683 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:39,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:39,778 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:40,233 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:40,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:41,282 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:16:41,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:41,622 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:41,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:42,234 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:42,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:43,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:44,235 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:44,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:45,438 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:45,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:46,235 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:46,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:47,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:48,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:49,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:50,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:50,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:51,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:52,238 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:52,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:53,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:54,238 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:54,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:55,638 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:55,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:56,239 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:56,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:56,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:56,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:57,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:58,240 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:58,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:59,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:00,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:01,442 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:01,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:02,241 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:02,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:03,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:04,242 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:04,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:05,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:06,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:06,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:06,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:07,690 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:08,079 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:17:08,079 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 with policy now +2024-12-11 19:17:08,082 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:17:08,082 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:08,082 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:08,083 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:08,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:08,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:08,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:17:08,353 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/q8qglt22-media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,660 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/ac716fp2-media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,691 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:09,694 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:10,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:10,696 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:11,282 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:17:11,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:11,629 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:11,688 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:11,715 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:12,246 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:12,699 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:13,692 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:14,695 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:15,697 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:16,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:17:16,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:16,525 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/6mzwuq1v-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:17:16,699 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:16,933 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:16,933 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:16,933 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:16,937 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:17,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:17,696 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:18,248 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:18,700 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:19,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:20,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:21,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:22,173 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:22,249 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:22,698 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:23,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:24,250 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:24,703 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:25,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:26,251 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:26,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:26,631 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:26,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:27,703 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:27,728 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:28,251 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:28,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:29,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:30,749 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:31,706 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:32,253 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:32,708 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:33,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:33,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:34,708 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:35,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:36,713 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:37,710 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:38,256 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:38,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:39,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:39,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:40,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:41,283 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:17:41,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:41,627 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:41,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:42,257 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:42,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:43,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:44,710 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:44,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:45,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:46,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:47,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:48,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:49,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:50,146 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:50,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:50,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:51,645 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:17:51,645 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 with policy now +2024-12-11 19:17:51,646 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:17:51,646 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:51,646 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:51,648 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:51,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:52,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:52,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:52,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:52,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:17:52,519 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/lxp6kvg9-media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:52,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:52,817 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/dmphh3x2-media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:53,726 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:54,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:54,727 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:55,538 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:55,733 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:56,262 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:56,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:56,634 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:56,734 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:57,750 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:58,262 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:58,757 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:59,752 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:59,983 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:59,983 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:59,984 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:00,263 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:00,263 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:00,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:00,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:01,753 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:02,264 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:02,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:03,745 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:04,752 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:05,751 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:06,660 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:06,749 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:07,750 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:08,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:09,760 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:10,755 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:11,283 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:18:11,639 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:18:11,640 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:18:11,753 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:11,754 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:12,754 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:13,758 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:14,755 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:15,762 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:16,006 DEBUG SenderThread:773446 [sender.py:send():379] send: exit +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():588] handling runtime: 274 +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():594] send defer +2024-12-11 19:18:16,006 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,006 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 19:18:16,007 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 1 +2024-12-11 19:18:16,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 19:18:16,007 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 2 +2024-12-11 19:18:16,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 19:18:16,007 DEBUG SystemMonitor:773446 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 19:18:16,007 DEBUG SystemMonitor:773446 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 19:18:16,008 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 19:18:16,063 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined network monitor +2024-12-11 19:18:16,064 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,064 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 19:18:16,064 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 3 +2024-12-11 19:18:16,064 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:18:16,064 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 19:18:16,065 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,065 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 19:18:16,065 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 4 +2024-12-11 19:18:16,065 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,065 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 19:18:16,268 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:18,269 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 19:18:22,985 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,986 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 19:18:22,986 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 5 +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:22,986 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:18:22,986 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send():379] send: summary +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 6 +2024-12-11 19:18:22,988 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,988 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 7 +2024-12-11 19:18:22,989 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,989 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,989 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 19:18:22,989 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,989 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 19:18:23,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:23,271 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:24,003 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 8 +2024-12-11 19:18:24,003 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:24,003 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:24,004 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 19:18:24,004 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:24,004 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 19:18:24,004 INFO SenderThread:773446 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 19:18:24,006 INFO SenderThread:773446 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 19:18:24,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:24,031 INFO SenderThread:773446 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 19:18:24,042 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 9 +2024-12-11 19:18:24,042 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:24,042 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:24,043 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 19:18:24,043 DEBUG SenderThread:773446 [sender.py:send():379] send: artifact +2024-12-11 19:18:24,271 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:24,605 INFO wandb-upload_2:773446 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpclrh1q5h +2024-12-11 19:18:24,605 INFO wandb-upload_0:773446 [upload_job.py:push():85] Skipped uploading /tmp/tmpnbwfrddq/wandb-job.json +2024-12-11 19:18:24,799 INFO wandb-upload_1:773446 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdrsc03bb +2024-12-11 19:18:25,008 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:25,398 INFO SenderThread:773446 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Nzk0NTgyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Nzk0MTU0Mw==', 'versionIndex': 370}}} +2024-12-11 19:18:25,398 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:25,398 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 19:18:25,398 INFO SenderThread:773446 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml config.yaml +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json wandb-summary.json +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt requirements.txt +2024-12-11 19:18:26,274 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json wandb-metadata.json +2024-12-11 19:18:26,274 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:26,279 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log output.log +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch diff.patch +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 10 +2024-12-11 19:18:26,281 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:26,281 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,281 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 19:18:26,284 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,284 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 19:18:26,284 INFO SenderThread:773446 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 19:18:26,455 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:26,481 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt +2024-12-11 19:18:26,523 INFO wandb-upload_4:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:26,524 INFO wandb-upload_2:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml +2024-12-11 19:18:26,535 INFO wandb-upload_3:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/e8bkegnr-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:26,735 INFO Thread-11 (_thread_body):773446 [sender.py:transition_state():614] send defer: 11 +2024-12-11 19:18:26,735 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,736 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 19:18:26,736 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,736 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 19:18:26,736 INFO SenderThread:773446 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 19:18:26,737 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 12 +2024-12-11 19:18:26,737 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,737 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 19:18:26,737 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,737 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 19:18:26,737 INFO SenderThread:773446 [file_stream.py:finish():601] file stream finish called +2024-12-11 19:18:26,972 INFO SenderThread:773446 [file_stream.py:finish():605] file stream finish is done +2024-12-11 19:18:26,972 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 13 +2024-12-11 19:18:26,973 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,973 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,973 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 19:18:26,973 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 14 +2024-12-11 19:18:26,973 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,973 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send():379] send: final +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send():379] send: footer +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,974 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 19:18:26,974 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:26,974 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:26,974 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 19:18:26,975 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: server_info +2024-12-11 19:18:26,976 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:26,976 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 19:18:26,977 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 19:18:26,977 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:27,035 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 19:18:27,036 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 19:18:27,036 INFO HandlerThread:773446 [handler.py:finish():882] shutting down handler +2024-12-11 19:18:27,976 INFO WriterThread:773446 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb +2024-12-11 19:18:28,036 INFO SenderThread:773446 [sender.py:finish():1615] shutting down sender +2024-12-11 19:18:28,036 INFO SenderThread:773446 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 19:18:28,036 INFO SenderThread:773446 [file_pusher.py:join():175] waiting for file pusher diff --git a/191334/wandb/debug.log b/191334/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..30d1487653f300de014d777f529ee92059972f0a --- /dev/null +++ b/191334/wandb/debug.log @@ -0,0 +1,31 @@ +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Configure stats pid to 773302 +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/settings +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug.log +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug-internal.log +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:init():569] calling init triggers +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 2, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '266406_0', 'wandb_notes': '266406_0_', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [4, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():619] starting backend +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():623] setting up manager +2024-12-11 19:13:40,901 INFO MainThread:773302 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 19:13:40,904 INFO MainThread:773302 [wandb_init.py:init():631] backend started and connected +2024-12-11 19:13:40,917 INFO MainThread:773302 [wandb_init.py:init():720] updated telemetry +2024-12-11 19:13:40,923 INFO MainThread:773302 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 19:13:41,191 INFO MainThread:773302 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 19:13:41,259 INFO MainThread:773302 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 19:13:41,261 INFO MainThread:773302 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 19:13:41,617 INFO MainThread:773302 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 19:13:41,618 INFO MainThread:773302 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 19:13:41,619 INFO MainThread:773302 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/tb, True +2024-12-11 19:13:45,500 INFO MainThread:773302 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 19:18:28,045 WARNING MsgRouterThr:773302 [router.py:message_loop():77] message_loop has been closed diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py b/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..75a95b02db44bf1d6cdce4329109f52a0c4c1bc8 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py @@ -0,0 +1,512 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml b/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05b8fcec93b07853a7b0b9630adced6adc06ad2d --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml @@ -0,0 +1,966 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 2 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '266406_0' +wandb_notes: + desc: null + value: '266406_0_' +eval: + desc: null + value: true +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: false +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: + - hbm_twodim_left_to_right +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 16 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + obs_shape: + - 4 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 + zero_centered: false + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +true_action_history: + desc: null + value: false +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1733962420 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 16 + - 23 + - 35 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch b/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..f10d8c9c6e861fb5ac963d165dc0588aa473f7c3 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch @@ -0,0 +1,57 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index 91f9f99..bb7c714 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -124,7 +124,17 @@ bc_weight_type: 'qfilter' # linear, qfilter + # Load weights + load_checkpoint: ${agent.load_checkpoint} + # 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +-# full dataset ++# full 32 demos ++# wandb_run_id: "265625" # RGBD history seed 0 ++# wandb_run_id: "266370_1" # RGBD history seed 1 ++# wandb_run_id: "266370_2" # RGBD history seed 2 ++# wandb_run_id: "266382_4" # RGBD history seed 4 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "266392_0" # RGBD+act history dataset seed 0 ++# wandb_run_id: "266407_0" # RGBD+act history dataset seed 1 ++wandb_run_id: "266406_0" # RGBD+act history dataset seed 2 ++ + # wandb_run_id: "265626" # RGBD+Mask+act history seed 0 + # wandb_run_id: "265638" # RGBD+Mask+act history seed 1 + # wandb_run_id: "265637" # RGBD+Mask+act history seed 2 +@@ -135,7 +145,32 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: "265630" # RGBD+Mask+contact+act history seed 1 + # wandb_run_id: "265631" # RGBD+Mask+contact+act history seed 2 + # wandb_run_id: "265633" # RGBD+Mask+contact+act history seed 3 +-wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++# wandb_run_id: "265634" # RGBD+Mask+contact+act history seed 4 ++ ++# 8 held out demos (26 demos) ++# wandb_run_id: "265652" # RGBD+Mask+contact+act history dataset seed 0 ++# wandb_run_id: "265654" # RGBD+Mask+contact+act history dataset seed 1 ++# wandb_run_id: "265655" # RGBD+Mask+contact+act history dataset seed 2 ++ ++# 8 held out demos (26 demos) ++# wandb_run_id: "265656" # RGBD+Mask+act history dataset seed 0 ++# wandb_run_id: "265657" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265658" # RGBD+Mask+act history dataset seed 2 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "265641" # RGBD+Mask+act history dataset seed 0 ++# wandb_run_id: "265644" # RGBD+Mask+act history dataset seed 1 ++# wandb_run_id: "265645" # RGBD+Mask+act history dataset seed 2 ++# wandb_run_id: "266381_0" # RGBD+Mask+act history dataset seed 4 ++ ++# 16 held out demos (16 demos) ++# wandb_run_id: "265646" # RGBD+Mask+Contact+act history dataset seed 0 ++# wandb_run_id: "265649" # RGBD+Mask+Contact+act history dataset seed 1 ++# wandb_run_id: "265650" # RGBD+Mask+Contact+act history dataset seed 2 ++# wandb_run_id: "266378_0" # RGBD+Mask+Contact+act history dataset seed 4 ++ ++ ++ + + # 54_240x320_hbm_twodim_fps_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # full dataset diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3734f776458717055cd3bb6c4fb63a4a4387c8a3 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da136561a2ad42749fdae27d6475d37bd6904f003bb617bd102700e75835318 +size 1151012 diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5433e8db927982af4f062c2ac2d023f00d001a57 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a45ace652f05385ee7562019925e1bc1406b8fdc00f0c2a8c0f3a6261051e5f +size 1151665 diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..390dec095718c79514dc3e2093a2d20721865b11 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9cccf7ec7cae45b2fa152dff13e74643ef89365bb489c5d108e4a154438176e +size 1239572 diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..45a1449b181c5e1d83cfdbb86b687a5eed1ee829 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334d1f02994a0fcdeda5419961508a7e674f6bbf2f7c4b38d27d336ee27676ec +size 1243778 diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f81363f0242e3532d1c60fb3aaff7e879fbd3dec --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35283caf4103c3829cd1d42553a2a9aa55f66bad96d4d9acf1eb68d56d1ef317 +size 1268903 diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log b/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..355df321ed2c6c396fe5ee9873bd42cc7e7f076c --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log @@ -0,0 +1,263 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1733962425.501528]: resetting environment +[INFO] [1733962425.503741]: cleared current plan +[INFO] [1733962425.504970]: moving to home +[INFO] [1733962426.507431]: reached home +[INFO] [1733962426.508574]: reset action history +[INFO] [1733962429.215545]: environment reset +Starting episode 0 +[INFO] [1733962429.217764]: resetting environment +[INFO] [1733962429.218818]: cleared current plan +[INFO] [1733962429.219946]: moving to home +[INFO] [1733962430.221876]: reached home +[INFO] [1733962430.223264]: reset action history +[INFO] [1733962432.930119]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1733962437.294333]: resetting environment +[INFO] [1733962437.296180]: cleared current plan +[INFO] [1733962437.298259]: moving to home +[INFO] [1733962438.300680]: reached home +[INFO] [1733962438.301981]: reset action history +[INFO] [1733962441.009679]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/episode_rosbags/episode_0_2024-12-11-19-14-01.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-773302.log +started roslaunch server http://158.130.50.37:46681/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [773570] +process[print_text-2]: started with pid [773571] +process[rosbag_record-3]: started with pid [773572] +started bagging! +For topic gripper_width: timestamp difference is -112948682 for nearest: 1733962447823948119 - target: 1733962447936896801 at idx 91 +For topic gripper_width: timestamp difference is -106565939 for nearest: 1733962455823925985 - target: 1733962455930491924 at idx 201 +For topic gripper_width: timestamp difference is -136166518 for nearest: 1733962460423930700 - target: 1733962460560097218 at idx 249 +[INFO] [1733962468.417341]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1733962469.138559]: Stored episode 1. +[INFO] [1733962469.139868]: Saving video... +[INFO] [1733962469.316209]: Video saved! +Starting episode 1 +[INFO] [1733962469.330705]: resetting environment +[INFO] [1733962469.331486]: cleared current plan +[INFO] [1733962469.331859]: moving to home +[INFO] [1733962472.136437]: reached home +[INFO] [1733962472.136898]: reset action history +[INFO] [1733962474.842866]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733962477.228348]: resetting environment +[INFO] [1733962477.229371]: cleared current plan +[INFO] [1733962477.230343]: moving to home +[INFO] [1733962478.232135]: reached home +[INFO] [1733962478.232627]: reset action history +[INFO] [1733962480.939046]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/episode_rosbags/episode_1_2024-12-11-19-14-40.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-773302.log +started roslaunch server http://158.130.50.37:34167/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [773676] +process[print_text-5]: started with pid [773700] +process[rosbag_record-6]: started with pid [773701] +started bagging! +For topic gripper_width: timestamp difference is -127159754 for nearest: 1733962485323950132 - target: 1733962485451109886 at idx 61 +For topic gripper_width: timestamp difference is -155548493 for nearest: 1733962487723923047 - target: 1733962487879471540 at idx 95 +For topic gripper_width: timestamp difference is -148979598 for nearest: 1733962490923924274 - target: 1733962491072903872 at idx 138 +For topic gripper_width: timestamp difference is -140346101 for nearest: 1733962495123938271 - target: 1733962495264284372 at idx 193 +For topic gripper_width: timestamp difference is -131602562 for nearest: 1733962507323947870 - target: 1733962507455550432 at idx 249 +[INFO] [1733962508.326133]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1733962509.085617]: Stored episode 2. +[INFO] [1733962509.086154]: Saving video... +[INFO] [1733962509.572251]: Video saved! +Starting episode 2 +[INFO] [1733962509.600026]: resetting environment +[INFO] [1733962509.602875]: cleared current plan +[INFO] [1733962509.604208]: moving to home +[INFO] [1733962512.508029]: reached home +[INFO] [1733962512.520455]: reset action history +[INFO] [1733962515.294815]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733962546.139461]: resetting environment +[INFO] [1733962546.141481]: cleared current plan +[INFO] [1733962546.145852]: moving to home +[INFO] [1733962547.149337]: reached home +[INFO] [1733962547.156180]: reset action history +[INFO] [1733962549.930298]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/episode_rosbags/episode_2_2024-12-11-19-15-49.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-773302.log +started roslaunch server http://158.130.50.37:36179/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [773830] +process[print_text-8]: started with pid [773831] +process[rosbag_record-9]: started with pid [773832] +started bagging! +For topic gripper_width: timestamp difference is -121712330 for nearest: 1733962560923941490 - target: 1733962561045653820 at idx 139 +For topic gripper_width: timestamp difference is -121584876 for nearest: 1733962561123918788 - target: 1733962561245503664 at idx 141 +For topic gripper_width: timestamp difference is -120972463 for nearest: 1733962561923882178 - target: 1733962562044854641 at idx 155 +For topic gripper_width: timestamp difference is -120303925 for nearest: 1733962562723903362 - target: 1733962562844207287 at idx 163 +For topic gripper_width: timestamp difference is -120126467 for nearest: 1733962562923921365 - target: 1733962563044047832 at idx 165 +For topic gripper_width: timestamp difference is -118874620 for nearest: 1733962564523924281 - target: 1733962564642798901 at idx 186 +For topic gripper_width: timestamp difference is -118712489 for nearest: 1733962564723919566 - target: 1733962564842632055 at idx 188 +For topic gripper_width: timestamp difference is -118241561 for nearest: 1733962565323890912 - target: 1733962565442132473 at idx 195 +For topic gripper_width: timestamp difference is -118048299 for nearest: 1733962565523924004 - target: 1733962565641972303 at idx 197 +For topic gripper_width: timestamp difference is -116245178 for nearest: 1733962567723945233 - target: 1733962567840190411 at idx 227 +For topic gripper_width: timestamp difference is -116117827 for nearest: 1733962567923913129 - target: 1733962568040030956 at idx 229 +For topic gripper_width: timestamp difference is -127329730 for nearest: 1733962569323927737 - target: 1733962569451257467 at idx 246 +For topic gripper_width: timestamp difference is -127051963 for nearest: 1733962569523921357 - target: 1733962569650973320 at idx 247 +For topic gripper_width: timestamp difference is -126704003 for nearest: 1733962569723874305 - target: 1733962569850578308 at idx 249 +For topic gripper_width: timestamp difference is -124603593 for nearest: 1733962570723923362 - target: 1733962570848526955 at idx 249 +For topic gripper_width: timestamp difference is -124237258 for nearest: 1733962570923874419 - target: 1733962571048111677 at idx 249 +For topic gripper_width: timestamp difference is -123743266 for nearest: 1733962571123941690 - target: 1733962571247684956 at idx 249 +For topic gripper_width: timestamp difference is -120467981 for nearest: 1733962572723877589 - target: 1733962572844345570 at idx 249 +For topic gripper_width: timestamp difference is -117876307 for nearest: 1733962573923932060 - target: 1733962574041808367 at idx 249 +For topic gripper_width: timestamp difference is -117449234 for nearest: 1733962574123925735 - target: 1733962574241374969 at idx 249 +For topic gripper_width: timestamp difference is -113736885 for nearest: 1733962575923909885 - target: 1733962576037646770 at idx 249 +For topic gripper_width: timestamp difference is -111240616 for nearest: 1733962577123951921 - target: 1733962577235192537 at idx 249 +For topic gripper_width: timestamp difference is -109266197 for nearest: 1733962578123926485 - target: 1733962578233192682 at idx 249 +For topic gripper_width: timestamp difference is -108846043 for nearest: 1733962578323920871 - target: 1733962578432766914 at idx 249 +For topic gripper_width: timestamp difference is -108465357 for nearest: 1733962578523910360 - target: 1733962578632375717 at idx 249 +[INFO] [1733962579.903526]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1733962580.680182]: Stored episode 3. +[INFO] [1733962580.682516]: Saving video... +[INFO] [1733962580.872804]: Video saved! +Starting episode 3 +[INFO] [1733962580.928169]: resetting environment +[INFO] [1733962580.932713]: cleared current plan +[INFO] [1733962580.936045]: moving to home +[INFO] [1733962583.645926]: reached home +[INFO] [1733962583.649049]: reset action history +[INFO] [1733962586.374455]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733962593.371608]: resetting environment +[INFO] [1733962593.377288]: cleared current plan +[INFO] [1733962593.381800]: moving to home +[INFO] [1733962594.385959]: reached home +[INFO] [1733962594.394051]: reset action history +[INFO] [1733962597.128479]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/episode_rosbags/episode_3_2024-12-11-19-16-37.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-773302.log +started roslaunch server http://158.130.50.37:45191/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [773960] +process[print_text-11]: started with pid [773961] +process[rosbag_record-12]: started with pid [773962] +started bagging! +For topic gripper_width: timestamp difference is -120847381 for nearest: 1733962603223920905 - target: 1733962603344768286 at idx 85 +For topic gripper_width: timestamp difference is -120070887 for nearest: 1733962604223920869 - target: 1733962604343991756 at idx 94 +For topic gripper_width: timestamp difference is -123615073 for nearest: 1733962611223919822 - target: 1733962611347534895 at idx 184 +For topic gripper_width: timestamp difference is -123341762 for nearest: 1733962613023922003 - target: 1733962613147263765 at idx 208 +For topic gripper_width: timestamp difference is -120412883 for nearest: 1733962614423919860 - target: 1733962614544332743 at idx 230 +For topic gripper_width: timestamp difference is -114774633 for nearest: 1733962617223870825 - target: 1733962617338645458 at idx 249 +For topic gripper_width: timestamp difference is -110255338 for nearest: 1733962619423944138 - target: 1733962619534199476 at idx 249 +For topic gripper_width: timestamp difference is -108657950 for nearest: 1733962620223950035 - target: 1733962620332607985 at idx 249 +For topic gripper_width: timestamp difference is -103375911 for nearest: 1733962622823929311 - target: 1733962622927305222 at idx 249 +For topic gripper_width: timestamp difference is -101717214 for nearest: 1733962623623920460 - target: 1733962623725637674 at idx 249 +[INFO] [1733962627.005235]: Storing episode... +[print_text-11] killing on exit +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[INFO] [1733962627.800546]: Stored episode 4. +[INFO] [1733962627.805152]: Saving video... +[INFO] [1733962628.022158]: Video saved! +Starting episode 4 +[INFO] [1733962628.082338]: resetting environment +[INFO] [1733962628.086639]: cleared current plan +[INFO] [1733962628.089920]: moving to home +[INFO] [1733962630.906424]: reached home +[INFO] [1733962630.909156]: reset action history +[INFO] [1733962633.755561]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1733962636.027454]: resetting environment +[INFO] [1733962636.031703]: cleared current plan +[INFO] [1733962636.037811]: moving to home +[INFO] [1733962637.049122]: reached home +[INFO] [1733962637.054624]: reset action history +[INFO] [1733962639.847857]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/episode_rosbags/episode_4_2024-12-11-19-17-19.bag +... logging to /home/leonmkim/.ros/log/1408e5ba-b74f-11ef-9a34-5546d6409a19/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-773302.log +started roslaunch server http://158.130.50.37:36633/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [774113] +process[print_text-14]: started with pid [774114] +process[rosbag_record-15]: started with pid [774115] +started bagging! +For topic gripper_width: timestamp difference is -126188727 for nearest: 1733962644923920659 - target: 1733962645050109386 at idx 75 +For topic gripper_width: timestamp difference is -124611001 for nearest: 1733962645723868508 - target: 1733962645848479509 at idx 85 +For topic gripper_width: timestamp difference is -114809845 for nearest: 1733962650523919966 - target: 1733962650638729811 at idx 143 +For topic gripper_width: timestamp difference is -110503214 for nearest: 1733962655323921425 - target: 1733962655434424639 at idx 207 +For topic gripper_width: timestamp difference is -111412838 for nearest: 1733962656123920843 - target: 1733962656235333681 at idx 216 +For topic gripper_width: timestamp difference is -108500294 for nearest: 1733962659723919817 - target: 1733962659832420111 at idx 249 +For topic gripper_width: timestamp difference is -134428791 for nearest: 1733962668923873850 - target: 1733962669058302641 at idx 249 +[INFO] [1733962670.522295]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1733962671.349140]: Stored episode 5. +[INFO] [1733962671.354508]: Saving video... +[INFO] [1733962671.589602]: Video saved! +[INFO] [1733962671.647246]: resetting environment +[INFO] [1733962671.655068]: cleared current plan +[INFO] [1733962671.661822]: moving to home +[INFO] [1733962674.483360]: reached home +[INFO] [1733962674.502695]: reset action history +[INFO] [1733962677.300227]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +proceeding to start episode! \ No newline at end of file diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt b/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..192179a195956e252bda33f2342b4970a9de79d9 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt @@ -0,0 +1,339 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.2 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json b/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a2a0df6006684c9690f4acc5049c48122738d459 --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2024-12-12T00:13:41.286723", + "startedAt": "2024-12-12T00:13:40.893574", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "1b557f164c7f62a8d1545c761b13ede1677c3a83" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.654375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.998, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.045, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 705.1167831420898 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71596908569336 + } +} diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json b/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..56ae70387bcbac8fee2d79fb1e907b8b05aa127e --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "3da136561a2ad42749fdae27d6475d37bd6904f003bb617bd102700e75835318", "size": 1151012, "path": "media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4"}, "global_step": 675, "_timestamp": 1733962679.691435, "_runtime": 258.78695702552795, "_step": 9, "eval/1_eval": {"_type": "video-file", "sha256": "3a45ace652f05385ee7562019925e1bc1406b8fdc00f0c2a8c0f3a6261051e5f", "size": 1151665, "path": "media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4"}, "eval/num_success": 5.0, "episode": 5.0, "eval/success_rate": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "c9cccf7ec7cae45b2fa152dff13e74643ef89365bb489c5d108e4a154438176e", "size": 1239572, "path": "media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "334d1f02994a0fcdeda5419961508a7e674f6bbf2f7c4b38d27d336ee27676ec", "size": 1243778, "path": "media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "35283caf4103c3829cd1d42553a2a9aa55f66bad96d4d9acf1eb68d56d1ef317", "size": 1268903, "path": "media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4"}, "_wandb": {"runtime": 274}} \ No newline at end of file diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug-internal.log b/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d7c220c3fc4b8b136df74bc6bbfb1f4b5bb50b4a --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug-internal.log @@ -0,0 +1,755 @@ +2024-12-11 19:13:40,902 INFO StreamThr :773446 [internal.py:wandb_internal():85] W&B internal server running at pid: 773446, started at: 2024-12-11 19:13:40.902318 +2024-12-11 19:13:40,903 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status +2024-12-11 19:13:40,918 INFO WriterThread:773446 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb +2024-12-11 19:13:40,918 DEBUG SenderThread:773446 [sender.py:send():379] send: header +2024-12-11 19:13:40,926 DEBUG SenderThread:773446 [sender.py:send():379] send: run +2024-12-11 19:13:41,175 INFO SenderThread:773446 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files +2024-12-11 19:13:41,175 INFO SenderThread:773446 [sender.py:_start_run_threads():1188] run started: l07wa4hd with start time 1733962420.904478 +2024-12-11 19:13:41,191 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 19:13:41,192 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: check_version +2024-12-11 19:13:41,268 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 19:13:41,273 DEBUG HandlerThread:773446 [system_info.py:__init__():26] System info init +2024-12-11 19:13:41,273 DEBUG HandlerThread:773446 [system_info.py:__init__():41] System info init done +2024-12-11 19:13:41,273 INFO HandlerThread:773446 [system_monitor.py:start():194] Starting system monitor +2024-12-11 19:13:41,273 INFO SystemMonitor:773446 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 19:13:41,273 INFO HandlerThread:773446 [system_monitor.py:probe():214] Collecting system info +2024-12-11 19:13:41,273 INFO SystemMonitor:773446 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 19:13:41,274 INFO SystemMonitor:773446 [interfaces.py:start():188] Started disk monitoring +2024-12-11 19:13:41,275 INFO SystemMonitor:773446 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 19:13:41,275 INFO SystemMonitor:773446 [interfaces.py:start():188] Started memory monitoring +2024-12-11 19:13:41,276 INFO SystemMonitor:773446 [interfaces.py:start():188] Started network monitoring +2024-12-11 19:13:41,286 DEBUG HandlerThread:773446 [system_info.py:probe():152] Probing system +2024-12-11 19:13:41,288 DEBUG HandlerThread:773446 [system_info.py:_probe_git():137] Probing git +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:_probe_git():145] Probing git done +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:probe():200] Probing system done +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-12T00:13:41.286723', 'startedAt': '2024-12-12T00:13:40.893574', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.654375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.998, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.045, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 705.1167831420898}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 19:13:41,293 INFO HandlerThread:773446 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 19:13:41,293 INFO HandlerThread:773446 [system_monitor.py:probe():227] Publishing system info +2024-12-11 19:13:41,293 DEBUG HandlerThread:773446 [system_info.py:_save_code():44] Saving code +2024-12-11 19:13:41,294 DEBUG HandlerThread:773446 [system_info.py:_save_code():67] Saving code done +2024-12-11 19:13:41,294 DEBUG HandlerThread:773446 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 19:13:41,362 DEBUG HandlerThread:773446 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 19:13:41,363 INFO HandlerThread:773446 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 19:13:41,366 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:13:41,366 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 19:13:41,366 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 19:13:41,367 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 19:13:41,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 19:13:41,616 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: python_packages +2024-12-11 19:13:41,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:13:41,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:13:41,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:41,619 INFO HandlerThread:773446 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/tb" + save: true +} +control { + relay_id: "85246c7cda7d49f9afbefde503c903d6" +} +_info { + stream_id: "l07wa4hd" +} + +2024-12-11 19:13:41,619 DEBUG HandlerThread:773446 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 19:13:41,639 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/r8wlspo2-wandb-metadata.json +2024-12-11 19:13:41,640 INFO wandb-upload_2:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/azg7zrgd-diff.patch +2024-12-11 19:13:41,683 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/3fipiwh0-code/FISH/eval_robot.py +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: telemetry +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: tbrecord +2024-12-11 19:13:41,705 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:13:41,705 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 with policy live +2024-12-11 19:13:41,892 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/77gm1cue-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:13:42,177 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:42,177 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code +2024-12-11 19:13:42,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH +2024-12-11 19:13:42,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:43,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:44,176 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:44,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:45,501 DEBUG SenderThread:773446 [sender.py:send():379] send: config +2024-12-11 19:13:45,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:46,507 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:46,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:47,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:48,178 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:48,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:49,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:50,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:51,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:51,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:52,179 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:52,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:53,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:54,179 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:13:54,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:55,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:56,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:13:56,616 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:13:56,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:56,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:13:57,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:58,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:13:59,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:00,181 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:00,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:01,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:02,182 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:02,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:02,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:03,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:04,182 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:04,619 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:05,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:06,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:07,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:08,011 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:08,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:09,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:10,184 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:10,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:11,616 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:11,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:11,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:12,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:13,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:13,745 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:14,185 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml +2024-12-11 19:14:14,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:15,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:16,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:17,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:18,186 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:18,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:19,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:19,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:20,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:21,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:22,187 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:22,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:23,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:24,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:24,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:25,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:26,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:26,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:26,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:27,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:28,623 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:29,326 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:14:29,327 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 with policy now +2024-12-11 19:14:29,327 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:14:29,328 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:14:29,328 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:14:29,329 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:14:29,595 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/p9oiypsh-media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:29,623 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos +2024-12-11 19:14:30,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media +2024-12-11 19:14:30,332 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:30,478 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/otzzej9i-media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:14:30,625 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:31,626 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:32,190 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:32,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:33,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:34,191 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:34,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:35,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:35,886 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:36,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:36,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:37,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:38,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:14:38,192 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:38,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:39,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:40,193 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:40,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:40,939 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:41,279 DEBUG SystemMonitor:773446 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 19:14:41,280 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:14:41,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:41,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:41,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:42,194 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:42,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:43,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:44,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:45,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:46,558 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:46,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:47,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:48,196 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:48,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:49,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:50,196 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:50,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:51,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:51,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:52,197 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:52,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:53,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:54,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:55,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:56,617 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:14:56,617 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:14:56,658 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:56,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:14:57,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:58,199 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:14:58,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:14:59,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:00,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:01,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:02,618 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:02,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:03,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:04,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:05,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:06,632 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:07,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:08,326 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:08,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:09,596 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:15:09,596 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 with policy now +2024-12-11 19:15:09,597 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:15:09,598 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:15:09,598 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:15:09,600 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:15:09,636 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:09,849 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/9qh203tq-media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,203 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:15:10,471 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/4acat9m8-media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:15:10,638 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:11,280 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:15:11,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:11,622 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:11,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:12,204 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:12,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:13,550 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:13,639 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:14,204 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:14,640 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:15,643 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:16,205 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:16,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:17,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:18,205 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:18,626 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:18,650 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:19,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:20,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:21,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:22,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:23,647 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:24,624 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:24,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:25,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:26,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:26,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:26,661 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:27,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:28,650 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:29,651 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:29,733 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:30,655 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:31,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:32,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:33,652 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:34,655 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:35,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:35,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:36,653 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:37,658 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:38,657 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:39,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:40,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:40,656 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:41,281 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:15:41,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:41,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:41,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:42,659 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:43,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:44,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:45,664 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:45,739 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:46,215 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:15:46,451 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/7wdb8k1w-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:15:46,670 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:47,040 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:15:47,040 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:15:47,040 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:15:47,215 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:15:47,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:48,216 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:48,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:49,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:50,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:50,978 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:51,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:52,217 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:52,670 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:53,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:54,218 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:15:54,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:55,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:56,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:15:56,620 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:15:56,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:56,737 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:15:57,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:58,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:15:59,665 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:00,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:01,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:02,087 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:02,220 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:02,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:03,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:04,221 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:04,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:05,666 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:06,222 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:06,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:07,628 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:07,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:08,222 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:08,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:09,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:10,223 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:10,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:11,281 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:16:11,620 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:11,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:11,667 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:12,224 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:12,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:12,691 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:13,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:14,224 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:14,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:15,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:16,225 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:16,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:17,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:18,226 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:18,286 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:18,668 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:19,669 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:20,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:20,673 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:20,921 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:16:20,921 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 with policy now +2024-12-11 19:16:20,922 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:16:20,922 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:16:20,922 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:16:20,923 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:16:21,212 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/wobof1o8-media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:16:21,502 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/a144xsb1-media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:16:21,676 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:22,227 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:22,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:23,633 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:23,693 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:24,675 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:25,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:26,229 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:26,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:26,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:26,672 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:27,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:28,229 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:28,681 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:28,689 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:29,688 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:30,689 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:31,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:32,682 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:33,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:33,879 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:16:33,882 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:16:33,882 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:33,883 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:16:34,231 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:16:34,231 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:16:34,232 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:34,683 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:35,685 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:36,232 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:36,681 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:37,684 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:38,233 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:38,683 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:39,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:39,778 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:40,233 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:40,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:41,282 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:16:41,622 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:41,622 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:41,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:42,234 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:42,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:43,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:44,235 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:44,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:45,438 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:45,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:46,235 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:46,677 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:47,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:48,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:49,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:50,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:50,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:51,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:52,238 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:52,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:53,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:54,238 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:54,678 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:55,638 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:16:55,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:56,239 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:56,621 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:16:56,621 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:16:56,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:57,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:58,240 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:16:58,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:16:59,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:00,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:01,442 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:01,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:02,241 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:02,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:03,679 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:04,242 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:04,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:05,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:06,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:06,630 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:06,680 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:07,690 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:08,079 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:17:08,079 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 with policy now +2024-12-11 19:17:08,082 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:17:08,082 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:08,082 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:08,083 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:08,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:08,244 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:08,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:17:08,353 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/q8qglt22-media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,660 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/ac716fp2-media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:17:08,691 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:09,694 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:10,245 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:10,696 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:11,282 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:17:11,629 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:11,629 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:11,688 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:11,715 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:12,246 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:12,699 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:13,692 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:14,695 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:15,697 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:16,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:17:16,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:16,525 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/6mzwuq1v-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:17:16,699 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:16,933 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:16,933 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:16,933 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:16,937 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:17,247 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:17,696 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:18,248 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:18,700 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:19,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:20,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:21,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:22,173 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:22,249 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:22,698 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:23,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:24,250 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:24,703 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:25,702 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:26,251 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:26,631 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:26,631 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:26,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:27,703 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:27,728 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:28,251 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:28,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:29,704 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:30,749 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:31,706 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:32,253 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:32,708 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:33,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:33,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:34,708 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:35,709 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:36,713 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:37,710 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:38,256 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:38,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:39,637 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:39,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:40,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:41,283 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:17:41,627 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:41,627 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:41,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:42,257 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:42,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:43,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:44,710 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:44,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:45,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:46,711 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:47,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:48,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:49,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:50,146 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:50,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:50,712 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:51,645 DEBUG SenderThread:773446 [sender.py:send():379] send: files +2024-12-11 19:17:51,645 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 with policy now +2024-12-11 19:17:51,646 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 19:17:51,646 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:51,646 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:51,648 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:17:51,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:52,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:17:52,260 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:52,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:52,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval +2024-12-11 19:17:52,519 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/lxp6kvg9-media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:52,722 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:52,817 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/dmphh3x2-media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:17:53,726 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:54,261 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:54,727 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:55,538 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:17:55,733 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:56,262 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:56,634 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:17:56,634 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:17:56,734 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:57,750 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:58,262 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:17:58,757 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:59,752 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:17:59,983 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:17:59,983 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:17:59,984 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:00,263 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:00,263 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:00,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:00,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:01,753 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:02,264 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:02,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:03,745 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:04,752 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:05,751 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:06,660 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:06,749 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:07,750 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:08,747 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:09,760 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:10,755 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:11,283 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:18:11,639 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 19:18:11,640 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: stop_status +2024-12-11 19:18:11,753 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:11,754 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:12,754 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:13,758 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:14,755 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:15,762 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:16,006 DEBUG SenderThread:773446 [sender.py:send():379] send: exit +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():588] handling runtime: 274 +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:16,006 INFO SenderThread:773446 [sender.py:send_exit():594] send defer +2024-12-11 19:18:16,006 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,006 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 19:18:16,007 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 1 +2024-12-11 19:18:16,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 19:18:16,007 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 19:18:16,007 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 2 +2024-12-11 19:18:16,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 19:18:16,007 DEBUG SystemMonitor:773446 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 19:18:16,007 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 19:18:16,007 DEBUG SystemMonitor:773446 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 19:18:16,008 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 19:18:16,063 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [interfaces.py:finish():200] Joined network monitor +2024-12-11 19:18:16,064 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,064 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 19:18:16,064 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 3 +2024-12-11 19:18:16,064 DEBUG SenderThread:773446 [sender.py:send():379] send: stats +2024-12-11 19:18:16,064 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,064 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 19:18:16,065 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:16,065 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 19:18:16,065 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 4 +2024-12-11 19:18:16,065 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:16,065 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 19:18:16,268 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:18,269 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 19:18:22,985 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,985 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,986 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 19:18:22,986 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 5 +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:22,986 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send():379] send: history +2024-12-11 19:18:22,986 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 19:18:22,986 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: summary_record +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send():379] send: summary +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 6 +2024-12-11 19:18:22,988 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,988 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 19:18:22,988 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 19:18:22,988 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 7 +2024-12-11 19:18:22,989 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 19:18:22,989 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:22,989 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 19:18:22,989 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:22,989 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 19:18:23,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:23,271 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:24,003 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 8 +2024-12-11 19:18:24,003 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:24,003 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:24,004 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 19:18:24,004 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:24,004 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 19:18:24,004 INFO SenderThread:773446 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 19:18:24,006 INFO SenderThread:773446 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 19:18:24,007 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:24,031 INFO SenderThread:773446 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 19:18:24,042 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 9 +2024-12-11 19:18:24,042 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:24,042 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:24,043 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 19:18:24,043 DEBUG SenderThread:773446 [sender.py:send():379] send: artifact +2024-12-11 19:18:24,271 INFO Thread-12 :773446 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:24,605 INFO wandb-upload_2:773446 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpclrh1q5h +2024-12-11 19:18:24,605 INFO wandb-upload_0:773446 [upload_job.py:push():85] Skipped uploading /tmp/tmpnbwfrddq/wandb-job.json +2024-12-11 19:18:24,799 INFO wandb-upload_1:773446 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdrsc03bb +2024-12-11 19:18:25,008 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:25,398 INFO SenderThread:773446 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2Nzk0NTgyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2Nzk0MTU0Mw==', 'versionIndex': 370}}} +2024-12-11 19:18:25,398 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:25,398 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 19:18:25,398 INFO SenderThread:773446 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml config.yaml +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json wandb-summary.json +2024-12-11 19:18:26,272 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt requirements.txt +2024-12-11 19:18:26,274 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-metadata.json wandb-metadata.json +2024-12-11 19:18:26,274 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:26,279 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log output.log +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/diff.patch diff.patch +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 media/videos/eval/3_eval_3_334d1f02994a0fcdeda5.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 media/videos/eval/4_eval_4_35283caf4103c3829cd1.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 media/videos/eval/2_eval_2_c9cccf7ec7cae45b2fa1.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 media/videos/eval/1_eval_1_3a45ace652f05385ee75.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 media/videos/eval/0_eval_0_3da136561a2ad42749fd.mp4 +2024-12-11 19:18:26,280 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 10 +2024-12-11 19:18:26,281 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:26,281 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,281 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 19:18:26,284 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,284 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 19:18:26,284 INFO SenderThread:773446 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 19:18:26,455 INFO wandb-upload_1:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/wandb-summary.json +2024-12-11 19:18:26,481 INFO wandb-upload_0:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/requirements.txt +2024-12-11 19:18:26,523 INFO wandb-upload_4:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/output.log +2024-12-11 19:18:26,524 INFO wandb-upload_2:773446 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/files/config.yaml +2024-12-11 19:18:26,535 INFO wandb-upload_3:773446 [upload_job.py:push():130] Uploaded file /tmp/tmpaa23qkokwandb/e8bkegnr-events.out.tfevents.1733962421.leonmkim-ROG-Strix-G15CS-G15CS.773302.0 +2024-12-11 19:18:26,735 INFO Thread-11 (_thread_body):773446 [sender.py:transition_state():614] send defer: 11 +2024-12-11 19:18:26,735 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,736 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 19:18:26,736 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,736 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 19:18:26,736 INFO SenderThread:773446 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 19:18:26,737 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 12 +2024-12-11 19:18:26,737 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,737 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 19:18:26,737 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,737 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 19:18:26,737 INFO SenderThread:773446 [file_stream.py:finish():601] file stream finish called +2024-12-11 19:18:26,972 INFO SenderThread:773446 [file_stream.py:finish():605] file stream finish is done +2024-12-11 19:18:26,972 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 13 +2024-12-11 19:18:26,973 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,973 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,973 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 19:18:26,973 INFO SenderThread:773446 [sender.py:transition_state():614] send defer: 14 +2024-12-11 19:18:26,973 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: defer +2024-12-11 19:18:26,973 INFO HandlerThread:773446 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send():379] send: final +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send():379] send: footer +2024-12-11 19:18:26,973 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: defer +2024-12-11 19:18:26,974 INFO SenderThread:773446 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 19:18:26,974 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:26,974 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:26,974 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 19:18:26,975 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: server_info +2024-12-11 19:18:26,976 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 19:18:26,976 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 19:18:26,977 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 19:18:26,977 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 19:18:27,035 DEBUG SenderThread:773446 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 19:18:27,036 INFO MainThread:773446 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 19:18:27,036 DEBUG HandlerThread:773446 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 19:18:27,036 INFO HandlerThread:773446 [handler.py:finish():882] shutting down handler +2024-12-11 19:18:27,976 INFO WriterThread:773446 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb +2024-12-11 19:18:28,036 INFO SenderThread:773446 [sender.py:finish():1615] shutting down sender +2024-12-11 19:18:28,036 INFO SenderThread:773446 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 19:18:28,036 INFO SenderThread:773446 [file_pusher.py:join():175] waiting for file pusher diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug.log b/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..30d1487653f300de014d777f529ee92059972f0a --- /dev/null +++ b/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug.log @@ -0,0 +1,31 @@ +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Configure stats pid to 773302 +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/settings +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug.log +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/wandb/run-20241211_191340-l07wa4hd/logs/debug-internal.log +2024-12-11 19:13:40,895 INFO MainThread:773302 [wandb_init.py:init():569] calling init triggers +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 2, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '266406_0', 'wandb_notes': '266406_0_', 'eval': True, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': False, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': ['hbm_twodim_left_to_right'], 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 16, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'obs_shape': [4, 240, 320], 'action_shape': [36, 7], 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}, 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'true_action_history': False, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt'} +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():619] starting backend +2024-12-11 19:13:40,896 INFO MainThread:773302 [wandb_init.py:init():623] setting up manager +2024-12-11 19:13:40,901 INFO MainThread:773302 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-12-11 19:13:40,904 INFO MainThread:773302 [wandb_init.py:init():631] backend started and connected +2024-12-11 19:13:40,917 INFO MainThread:773302 [wandb_init.py:init():720] updated telemetry +2024-12-11 19:13:40,923 INFO MainThread:773302 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-12-11 19:13:41,191 INFO MainThread:773302 [wandb_run.py:_on_init():2435] communicating current version +2024-12-11 19:13:41,259 INFO MainThread:773302 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.0 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-12-11 19:13:41,261 INFO MainThread:773302 [wandb_init.py:init():804] starting run threads in backend +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_console_start():2413] atexit reg +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2024-12-11 19:13:41,616 INFO MainThread:773302 [wandb_run.py:_redirect():2320] Wrapping output streams. +2024-12-11 19:13:41,617 INFO MainThread:773302 [wandb_run.py:_redirect():2345] Redirects installed. +2024-12-11 19:13:41,618 INFO MainThread:773302 [wandb_init.py:init():847] run started, returning control to user process +2024-12-11 19:13:41,619 INFO MainThread:773302 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/266406_0/191334/tb, True +2024-12-11 19:13:45,500 INFO MainThread:773302 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2024-12-11 19:18:28,045 WARNING MsgRouterThr:773302 [router.py:message_loop():77] message_loop has been closed diff --git a/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb b/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4d5a3e9b3a8c6ee67f4b12d2b673d63533a6cb3f Binary files /dev/null and b/191334/wandb/run-20241211_191340-l07wa4hd/run-l07wa4hd.wandb differ diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c2ff3b934e9a68655a971c3b1365afd5359986a --- /dev/null +++ b/config.yaml @@ -0,0 +1,453 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 0 +dataset_shuffle_seed: 2 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '266406_0' +wandb_notes: '266406_0_' +eval: false +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: +- hbm_twodim_left_to_right +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 64_240x320_hbm_twodim_fps_fix_contact_fix_recovery_fix_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 16 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + obs_shape: + - 4 + - 240 + - 320 + action_shape: + - 36 + - 7 + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/snapshot_10499.pt b/snapshot_10499.pt new file mode 100644 index 0000000000000000000000000000000000000000..423d6a39b64c6b32cc79d24ea756321d42dd7c5c --- /dev/null +++ b/snapshot_10499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1702c03ee954b0d8a50d79858fc6fece9df5efb19a636c080069d8c2e4773efd +size 910064728 diff --git a/snapshot_10500.pt b/snapshot_10500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9e17ea58162401c80f2018e530a65416642e691 --- /dev/null +++ b/snapshot_10500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a1c795bfd12b3845aaa2bc4d017e14a0124ac8ac4b9b084e8e81a04572f18c4 +size 910064728 diff --git a/snapshot_11249.pt b/snapshot_11249.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5d296d78debbb2b3082f22683bffb3001e90184 --- /dev/null +++ b/snapshot_11249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a7baefb5d72aed8aa4c0221b74be04ee442af7d1f08e52d025e49698fc7e47 +size 910064728 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea040880ca1b23bd10f97d197e1c3c6c488704c4 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60ce39abcf5b30ca6e95fff78604d99d31fafd40899cc3d130204247b33b5b47 +size 910064728 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae1a876f59561e93e26eb18a854b006f1c0a6d72 --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea1cc29c54827d1659ec1aad224b07479e91851cdb2c1c01725d036b3c2e3bd4 +size 910064728 diff --git a/snapshot_14999.pt b/snapshot_14999.pt new file mode 100644 index 0000000000000000000000000000000000000000..268f303ad665cb18770381e8e42476755fbda59f --- /dev/null +++ b/snapshot_14999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d69e4bbea743e8f03fa03fa80f77dadf62de8183235751592ed02e3e4ef007ca +size 910064728 diff --git a/snapshot_1500.pt b/snapshot_1500.pt new file mode 100644 index 0000000000000000000000000000000000000000..c142df85fd96aacd88ae4d90708db689083279f6 --- /dev/null +++ b/snapshot_1500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01ac6c06e998632bf1c3b84fbab14b80411d3095aef23b50df942006dc1518ec +size 910064728 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cc097486f0fbaeea9c16b3e4c5f33e9c6aa42a7 --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8f47ca3ad72d50abd7429def6a2b45051fbc466da722cd7995e80f622d8356 +size 910064728 diff --git a/snapshot_3000.pt b/snapshot_3000.pt new file mode 100644 index 0000000000000000000000000000000000000000..27f690247670ba8b5be95681b7f32cefb4bf3ce6 --- /dev/null +++ b/snapshot_3000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b86fa1c25822d75cf51a274aaaeb1fb5211b3fa6945d971a48a826e55b8d0ae +size 910064728 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbe0e1ad1373c019ac6bd06cb5578c50d679e999 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59e7f4190b4f0d01631551bcb0ed2de6c9fcdbf88699eca8b019c68917d7c55c +size 910064728 diff --git a/snapshot_5749.pt b/snapshot_5749.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bba3e24f1febfdd59f584a66bf33376f468ec26 --- /dev/null +++ b/snapshot_5749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00bf7d6778c3be0fe06abe57fc957dae8ca1fca70ac1ec6016faf9e0a785630d +size 910064728 diff --git a/snapshot_6000.pt b/snapshot_6000.pt new file mode 100644 index 0000000000000000000000000000000000000000..68bbc655cc64759eeb1ce0dce09fa5599424bdf5 --- /dev/null +++ b/snapshot_6000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a962749e3bdcc2471d37b7f6927f2738c212fbb6725029e9995b451d07c1fd4 +size 910064728 diff --git a/snapshot_6499.pt b/snapshot_6499.pt new file mode 100644 index 0000000000000000000000000000000000000000..933398ea05a0c17625a386b405c2de5d7b48710d --- /dev/null +++ b/snapshot_6499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b92fd636b8db67bf5334fa84436471ef7ef76c19463ef9a847ebfcd69412b7e8 +size 910064728 diff --git a/snapshot_7500.pt b/snapshot_7500.pt new file mode 100644 index 0000000000000000000000000000000000000000..88832c54b92a29ea97fce16cbd588789ba584081 --- /dev/null +++ b/snapshot_7500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91829740d1059bf22e2f3eaa54c99b73dcc318a5e2b8fc954ba99ea214d8da84 +size 910064728 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..b50deeb3c0372b60dd489a49008684d56c7435df --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2990cdab91f2e112918aacb18d53fcdba4a6842de7dafd8d69541aeb280dbd6e +size 910064728 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cc097486f0fbaeea9c16b3e4c5f33e9c6aa42a7 --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc8f47ca3ad72d50abd7429def6a2b45051fbc466da722cd7995e80f622d8356 +size 910064728